@storacha/clawracha 0.0.4 → 0.0.5

package/src/sync.ts

@@ -1,258 +0,0 @@
-/**
- * Sync engine - orchestrates the full sync loop
- *
- * 1. Watch for file changes
- * 2. Encode changed files to root CIDs
- * 3. Diff local vs pail entries → put/del ops
- * 4. Generate UCN revision via batch (puts) + individual ops (dels)
- * 5. Upload all blocks as CAR
- * 6. Apply remote changes to local filesystem
- */
-
-import { CID } from "multiformats/cid";
-// UCN Pail imports
-import { Agent, Name, NoValueError, Revision } from "@storacha/ucn/pail";
-import type { NameView, ValueView } from "@storacha/ucn/pail/api";
-
-import type {
-  SyncState,
-  FileChange,
-  PailOp,
-  DeviceConfig,
-} from "./types/index.js";
-import type { Block } from "multiformats";
-import {
-  createWorkspaceBlockstore,
-  type WorkspaceBlockstore,
-} from "./blockstore/index.js";
-import { applyPendingOps } from "./handlers/apply.js";
-import { applyRemoteChanges } from "./handlers/remote.js";
-import { processChanges } from "./handlers/process.js";
-import { diffRemoteChanges, type PailEntries } from "./utils/differ.js";
-import { WritableCar, makeTempCar } from "./utils/tempcar.js";
-import { Client } from "@storacha/client";
-
-export class SyncEngine {
-  private workspace: string;
-  private blocks: WorkspaceBlockstore;
-  private name: NameView | null = null;
-  private current: ValueView | null = null;
-  private pendingOps: PailOp[] = [];
-  private carFile: WritableCar | null = null;
-  private running = false;
-  private lastSync: number | null = null;
-  private storachaClient: Client;
-
-  constructor(storachaClient: Client, workspace: string) {
-    this.storachaClient = storachaClient;
-    this.workspace = workspace;
-    this.blocks = createWorkspaceBlockstore(workspace);
-  }
-
-  /**
-   * Initialize sync engine with device config
-   */
-  async init(config: DeviceConfig): Promise<void> {
-    const agent = Agent.parse(config.agentKey);
-
-    if (config.nameArchive) {
-      // Restore from previously saved archive (has full state)
-      const archiveBytes = Buffer.from(config.nameArchive, "base64");
-      this.name = await Name.extract(agent, archiveBytes);
-    } else if (config.nameDelegation) {
-      // Reconstruct from delegation (granted by another device)
-      const { extract } = await import("@storacha/client/delegation");
-      const nameBytes = Buffer.from(config.nameDelegation, "base64");
-      const { ok: delegation } = await extract(nameBytes);
-      if (!delegation) throw new Error("Failed to extract name delegation");
-      this.name = Name.from(agent, [delegation]);
-    } else {
-      // First device — create a new name
-      this.name = await Name.create(agent);
-    }
-
-    try {
-      const result = await Revision.resolve(this.blocks, this.name);
-      this.current = result.value;
-      await this.storeBlocks(result.additions);
-    } catch (err) {
-      if (err instanceof NoValueError) {
-        this.current = null;
-      } else {
-        throw err;
-      }
-    }
-  }
-
-  /**
-   * Process a batch of file changes
-   */
-  async processChanges(changes: FileChange[]): Promise<void> {
-    if (!this.carFile) {
-      this.carFile = await makeTempCar();
-    }
-    const pendingOps = await processChanges(
-      changes,
-      this.workspace,
-      this.current,
-      this.blocks,
-      (block) => this.carFile!.put(block),
-      (block) => this.blocks.put(block),
-    );
-    this.pendingOps.push(...pendingOps);
-  }
-
-  /**
-   * Execute sync: generate revision, publish, upload, apply remote changes
-   */
-  async sync(): Promise<void> {
-    if (!this.name) {
-      throw new Error("Sync engine not initialized");
-    }
-
-    const beforeEntries = await this.getPailEntries();
-
-    if (this.pendingOps.length > 0) {
-      if (!this.carFile) {
-        throw new Error("CAR file not initialized");
-      }
-      const result = await applyPendingOps(
-        this.blocks,
-        this.name,
-        this.current,
-        this.pendingOps,
-      );
-      this.current = result.current;
-      for (const block of result.revisionBlocks) {
-        await this.carFile.put(block);
-      }
-      await this.storeBlocks(result.revisionBlocks);
-    } else {
-      // No pending ops — just pull remote
-      try {
-        const result = await Revision.resolve(this.blocks, this.name, {
-          base: this.current ?? undefined,
-        });
-        await this.storeBlocks(result.additions);
-        this.current = result.value;
-      } catch (err) {
-        if (!(err instanceof NoValueError)) throw err;
-      }
-    }
-
-    this.pendingOps = [];
-
-    // Upload all accumulated blocks as CAR
-    await this.possiblyUploadCAR();
-
-    // Apply remote changes
-    const afterEntries = await this.getPailEntries();
-    const remoteChanges = diffRemoteChanges(beforeEntries, afterEntries);
-    if (remoteChanges.length > 0) {
-      await this.applyRemoteChanges(remoteChanges, afterEntries);
-    }
-
-    this.lastSync = Date.now();
-  }
-
-  /**
-   * Create CAR and upload to Storacha
-   */
-  private async possiblyUploadCAR(): Promise<void> {
-    if (this.carFile) {
-      const readableCar = await this.carFile.switchToReadable();
-      this.carFile = null;
-      if (readableCar) {
-        try {
-          await this.storachaClient.uploadCAR(readableCar.readable);
-        } finally {
-          await readableCar.cleanup();
-        }
-      }
-    }
-  }
-
-  /**
-   * Get current pail entries as map
-   */
-  async getPailEntries(): Promise<PailEntries> {
-    const entries: PailEntries = new Map();
-    if (!this.current) return entries;
-
-    for await (const [key, value] of Revision.entries(
-      this.blocks,
-      this.current,
-    )) {
-      const cid = value as CID;
-      if (cid) entries.set(key, cid);
-    }
-    return entries;
-  }
-
-  /**
-   * Apply remote changes to local filesystem
-   */
-  private async applyRemoteChanges(
-    changedPaths: string[],
-    entries: PailEntries,
-  ): Promise<void> {
-    await applyRemoteChanges(changedPaths, entries, this.workspace, {
-      blocks: this.blocks,
-      current: this.current ?? undefined,
-    });
-  }
-
-
-  /**
-   * Pull all remote state and write to local filesystem.
-   * Used by /storacha-join to overwrite local with remote before watcher starts.
-   */
-  async pullRemote(): Promise<number> {
-    if (!this.name) throw new Error("Sync engine not initialized");
-
-    try {
-      const result = await Revision.resolve(this.blocks, this.name, {
-        base: this.current ?? undefined,
-      });
-      await this.storeBlocks(result.additions);
-      this.current = result.value;
-    } catch (err) {
-      if (!(err instanceof NoValueError)) throw err;
-    }
-
-    const entries = await this.getPailEntries();
-    if (entries.size > 0) {
-      const allPaths = [...entries.keys()];
-      await applyRemoteChanges(allPaths, entries, this.workspace, {
-        blocks: this.blocks,
-        current: this.current ?? undefined,
-      });
-    }
-
-    this.lastSync = Date.now();
-    return entries.size;
-  }
-
-  async status(): Promise<SyncState> {
-    const entries = await this.getPailEntries();
-    return {
-      running: this.running,
-      lastSync: this.lastSync,
-      root: this.current?.root ?? null,
-      entryCount: entries.size,
-      pendingChanges: this.pendingOps.length,
-    };
-  }
-
-  async exportNameArchive(): Promise<string> {
-    if (!this.name) throw new Error("Sync engine not initialized");
-    const bytes = await this.name.archive();
-    return Buffer.from(bytes).toString("base64");
-  }
-
-  private async storeBlocks(blocks: Block[]): Promise<void> {
-    for (const block of blocks) {
-      await this.blocks.put(block);
-    }
-  }
-}

package/src/types/index.ts

@@ -1,64 +0,0 @@
-/**
- * Type definitions for Storacha workspace sync
- */
-
-import type { UnknownLink } from "multiformats";
-import type { Block } from "multiformats";
-import type { CID } from "multiformats/cid";
-
-/** Plugin configuration (from openclaw.plugin.json schema) */
-export interface SyncPluginConfig {
-  enabled: boolean;
-  watchPatterns: string[];
-  ignorePatterns: string[];
-}
-
-/** Stored in .storacha/config.json — device-specific, not synced */
-export interface DeviceConfig {
-  /** Ed25519 agent private key (base64) */
-  agentKey: string;
-  /** UCN name archive (base64 CAR) */
-  nameArchive?: string;
-  /** Space → agent upload delegation (base64 archive) */
-  uploadDelegation?: string;
-  /** Name → agent delegation for pail sync (base64 archive) */
-  nameDelegation?: string;
-  /** Space DID extracted from upload delegation */
-  spaceDID?: string;
-  /** Whether setup is complete (watcher won't start without this) */
-  setupComplete?: boolean;
-}
-
-/** Current sync state */
-export interface SyncState {
-  /** Is sync currently running? */
-  running: boolean;
-  /** Last successful sync timestamp */
-  lastSync: number | null;
-  /** Current pail root CID */
-  root: UnknownLink | null;
-  /** Number of tracked entries */
-  entryCount: number;
-  /** Pending local changes not yet pushed */
-  pendingChanges: number;
-}
-
-/** File change event from watcher */
-export interface FileChange {
-  type: "add" | "change" | "unlink";
-  path: string; // relative to workspace
-}
-
-/** Encoded file result */
-export interface EncodedFile {
-  path: string;
-  blocks: ReadableStream<Block>;
-  size: number;
-}
-
-/** Diff operation for pail */
-export interface PailOp {
-  type: "put" | "del";
-  key: string; // file path
-  value?: CID; // root CID for puts
-}

package/src/utils/client.ts

@@ -1,51 +0,0 @@
-/**
- * Create a @storacha/client instance from an agent private key and upload delegation.
- *
- * Uses the top-level `create()` factory which accepts principal + store directly,
- * avoiding internal @storacha/access imports.
- */
-
-import { Agent as PailAgent } from "@storacha/ucn/pail";
-import { create as createClient } from "@storacha/client";
-import { StoreMemory } from "@storacha/client/stores/memory";
-import { extract } from "@storacha/client/delegation";
-import type { DeviceConfig } from "../types/index.js";
-import type { Client } from "@storacha/client";
-
-/**
- * Build a Storacha Client from device config.
- * Requires agentKey and uploadDelegation to be present.
- */
-export async function createStorachaClient(
-  config: DeviceConfig,
-): Promise<Client> {
-  if (!config.uploadDelegation) {
-    throw new Error("No upload delegation in device config");
-  }
-
-  const agent = PailAgent.parse(config.agentKey);
-
-  const client = await createClient({
-    principal: agent,
-    store: new StoreMemory(),
-  });
-
-  // Import the upload delegation (space → agent)
-  const uploadBytes = Buffer.from(config.uploadDelegation, "base64");
-  const { ok: uploadDelegation, error: uploadErr } =
-    await extract(uploadBytes);
-  if (!uploadDelegation) {
-    throw new Error(`Failed to extract upload delegation: ${uploadErr}`);
-  }
-
-  // addSpace registers the space and adds the delegation as a proof
-  await client.addSpace(uploadDelegation);
-
-  // Set the space as current so uploads target it
-  const spaceDID = uploadDelegation.capabilities[0]?.with;
-  if (spaceDID) {
-    await client.setCurrentSpace(spaceDID as `did:key:${string}`);
-  }
-
-  return client;
-}

package/src/utils/differ.ts

@@ -1,67 +0,0 @@
-/**
- * Differ - compares local directory tree with pail entries
- *
- * Generates put/del operations to sync local state to pail.
- */
-
-import type { CID } from "multiformats/cid";
-import type { PailOp } from "../types/index.js";
-
-/** Map of path → CID from pail entries */
-export type PailEntries = Map<string, CID>;
-
-/** Map of path → CID from local encoded files */
-export type LocalEntries = Map<string, CID>;
-
-/**
- * Compute diff between local files and pail entries
- *
- * @param local - Encoded local files (path → rootCID)
- * @param pail - Current pail entries (path → CID)
- * @returns Operations to apply to pail
- */
-export function diffEntries(local: LocalEntries, pail: PailEntries): PailOp[] {
-  const ops: PailOp[] = [];
-
-  // Find puts: files in local that are new or changed
-  for (const [path, localCID] of local) {
-    const pailCID = pail.get(path);
-    if (!pailCID || !localCID.equals(pailCID)) {
-      ops.push({ type: "put", key: path, value: localCID });
-    }
-  }
-
-  // Find deletes: files in pail that aren't in local
-  for (const path of pail.keys()) {
-    if (!local.has(path)) {
-      ops.push({ type: "del", key: path });
-    }
-  }
-
-  return ops;
-}
-
-
-/**
- * Diff two pail states to find files that changed remotely
- * (Used after publish to determine what to download)
- *
- * @param before - Pail entries before publish
- * @param after - Pail entries after publish (may include remote changes)
- * @returns Paths that changed remotely (need to download)
- */
-export function diffRemoteChanges(
-  before: PailEntries,
-  after: PailEntries
-): string[] {
-  const changed: string[] = [];
-
-  for (const [path, afterCID] of after) {
-    const beforeCID = before.get(path);
-    if (!beforeCID || !afterCID.equals(beforeCID)) {
-      changed.push(path);
-    }
-  }
-
-  return changed;
-}

package/src/utils/encoder.ts

@@ -1,64 +0,0 @@
-/**
- * File encoder - converts files to UnixFS DAG with root CID
- *
- * Uses @storacha/upload-client's UnixFS encoding to generate
- * content-addressed blocks for each file.
- */
-
-import * as fs from "node:fs/promises";
-import * as path from "node:path";
-import * as stream from "node:stream";
-import { createFileEncoderStream } from "@storacha/upload-client/unixfs";
-import type { EncodedFile } from "../types/index.js";
-
-/**
- * Encode a single file to UnixFS blocks
- */
-export async function encodeWorkspaceFile(
-  workspacePath: string,
-  relativePath: string,
-): Promise<EncodedFile> {
-  const fullPath = path.join(workspacePath, relativePath);
-  const fileHandle = await fs.open(fullPath);
-  const stat = await fs.stat(fullPath);
-
-  // Encode to UnixFS - returns { cid, blocks }
-  const blocks = createFileEncoderStream({
-    // @ts-expect-error node web stream not type compatible with web stream
-    stream() {
-      return stream.Readable.toWeb(fileHandle.createReadStream());
-    },
-  });
-
-  return {
-    path: relativePath,
-    size: stat.size,
-    blocks: blocks as any,
-  };
-}
-
-/**
- * Encode multiple files, returning all encoded results
- */
-export async function encodeFiles(
-  workspacePath: string,
-  relativePaths: string[],
-): Promise<EncodedFile[]> {
-  const results: EncodedFile[] = [];
-
-  for (const relativePath of relativePaths) {
-    try {
-      const encoded = await encodeWorkspaceFile(workspacePath, relativePath);
-      results.push(encoded);
-    } catch (err: any) {
-      if (err.code === "ENOENT") {
-        // File was deleted between detection and encoding - skip
-        console.warn(`File not found during encoding: ${relativePath}`);
-        continue;
-      }
-      throw err;
-    }
-  }
-
-  return results;
-}

package/src/utils/tempcar.ts

@@ -1,79 +0,0 @@
-import * as fs from "node:fs/promises";
-import { createReadStream, createWriteStream } from "node:fs";
-import * as os from "node:os";
-import * as path from "node:path";
-import * as stream from "node:stream";
-import type { Block } from "multiformats";
-import { CarWriter } from "@ipld/car/writer";
-import type { BlobLike } from "@storacha/client/types";
-
-export interface WritableCar {
-  /**
-   * Write a block to the CAR file.
-   * Must not be called after close/switchToReadable.
-   */
-  put(block: Block): Promise<void>;
-  cleanup(): Promise<void>;
-  /**
-   * Close the writer and return a readable for upload.
-   * Returns null if nothing was written.
-   */
-  switchToReadable(): Promise<ReadableCar | null>;
-}
-
-interface ReadableCar {
-  readable: BlobLike;
-  cleanup(): Promise<void>;
-}
-
-export const makeTempCar = async (): Promise<WritableCar> => {
-  const dir = await fs.mkdtemp(path.join(os.tmpdir(), "sync-clawracha-"));
-  const cleanup = async () => await fs.rm(dir, { recursive: true });
-  const file = path.join(dir, "blocks.car");
-
-  // CarWriter.createAppender() creates a headerless/rootless CAR
-  // which is fine for uploadCAR (the client handles it)
-  const { writer, out } = CarWriter.createAppender();
-
-  // Pipe the output to a file
-  const fsWriteStream = createWriteStream(file);
-  const pipePromise = stream.promises.pipeline(
-    stream.Readable.from(out),
-    fsWriteStream,
-  );
-
-  let didWrite = false;
-
-  const put = async (block: Block) => {
-    didWrite = true;
-    await writer.put({ cid: block.cid as any, bytes: block.bytes });
-  };
-
-  const switchToReadable = async (): Promise<ReadableCar | null> => {
-    try {
-      await writer.close();
-      await pipePromise;
-
-      if (!didWrite) {
-        await cleanup();
-        return null;
-      }
-
-      return {
-        readable: {
-          stream: () =>
-            stream.Readable.toWeb(createReadStream(file)) as unknown as ReadableStream<
-              Uint8Array<ArrayBuffer>
-            >,
-        },
-        cleanup,
-      };
-    } catch (err) {
-      console.error("Error switching to readable:", err);
-      await cleanup();
-      throw err;
-    }
-  };
-
-  return { put, switchToReadable, cleanup };
-};