@push.rocks/containerarchive 0.0.2

package/license

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2026 Lossless GmbH
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/npmextra.json

@@ -0,0 +1,40 @@
+{
+  "@git.zone/cli": {
+    "projectType": "npm",
+    "module": {
+      "githost": "code.foss.global",
+      "gitscope": "push.rocks",
+      "gitrepo": "containerarchive",
+      "description": "content-addressed incremental backup engine with deduplication, encryption, and error correction",
+      "npmPackagename": "@push.rocks/containerarchive",
+      "license": "MIT",
+      "keywords": [
+        "backup",
+        "deduplication",
+        "content-addressed",
+        "incremental",
+        "archive",
+        "encryption",
+        "chunking",
+        "fastcdc",
+        "pack-files"
+      ]
+    },
+    "release": {
+      "registries": [
+        "https://verdaccio.lossless.digital",
+        "https://registry.npmjs.org"
+      ],
+      "accessLevel": "public"
+    }
+  },
+  "@git.zone/tsrust": {
+    "targets": [
+      "linux_amd64",
+      "linux_arm64"
+    ]
+  },
+  "@git.zone/tsdoc": {
+    "legal": "\n## License and Legal Information\n\nThis module is part of the @push.rocks ecosystem, maintained by Task Venture Capital GmbH.\n\nLicensed under MIT. See LICENSE file for details.\n\nFor questions or commercial licensing, contact: hello@task.vc\n"
+  }
+}

package/package.json

@@ -0,0 +1,59 @@
+{
+  "name": "@push.rocks/containerarchive",
+  "version": "0.0.2",
+  "private": false,
+  "description": "content-addressed incremental backup engine with deduplication, encryption, and error correction",
+  "main": "dist_ts/index.js",
+  "typings": "dist_ts/index.d.ts",
+  "type": "module",
+  "scripts": {
+    "test": "(tstest test/ --verbose --timeout 60)",
+    "build": "(tsrust && tsbuild tsfolders --allowimplicitany)"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://code.foss.global/push.rocks/containerarchive.git"
+  },
+  "author": "Lossless GmbH",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://code.foss.global/push.rocks/containerarchive/issues"
+  },
+  "homepage": "https://code.foss.global/push.rocks/containerarchive",
+  "dependencies": {
+    "@push.rocks/lik": "^6.0.0",
+    "@push.rocks/smartpromise": "^4.0.0",
+    "@push.rocks/smartrust": "^1.3.2",
+    "@push.rocks/smartrx": "^3.0.0"
+  },
+  "devDependencies": {
+    "@git.zone/tsbuild": "^2.0.0",
+    "@git.zone/tsrun": "^1.0.0",
+    "@git.zone/tstest": "^1.0.0",
+    "@git.zone/tsrust": "^1.3.0",
+    "@types/node": "^22.0.0"
+  },
+  "files": [
+    "ts/**/*",
+    "dist/**/*",
+    "dist_*/**/*",
+    "dist_ts/**/*",
+    "assets/**/*",
+    "npmextra.json",
+    "readme.md"
+  ],
+  "browserslist": [
+    "last 1 chrome versions"
+  ],
+  "keywords": [
+    "backup",
+    "deduplication",
+    "content-addressed",
+    "incremental",
+    "archive",
+    "encryption",
+    "chunking",
+    "fastcdc",
+    "pack-files"
+  ]
+}

package/ts/00_commitinfo_data.ts

@@ -0,0 +1,8 @@
+/**
+ * autocreated commitinfo by @push.rocks/commitinfo
+ */
+export const commitinfo = {
+  name: '@push.rocks/containerarchive',
+  version: '0.0.2',
+  description: 'content-addressed incremental backup engine with deduplication, encryption, and error correction'
+}

package/ts/classes.containerarchive.ts

@@ -0,0 +1,419 @@
+import * as plugins from './plugins.js';
+import { commitinfo } from './00_commitinfo_data.js';
+import type {
+  TContainerArchiveCommands,
+  IInitOptions,
+  IOpenOptions,
+  IIngestOptions,
+  IIngestItem,
+  IIngestItemOptions,
+  IRestoreOptions,
+  ISnapshot,
+  ISnapshotFilter,
+  IVerifyOptions,
+  IVerifyResult,
+  IRetentionPolicy,
+  IPruneResult,
+  IRepairResult,
+  IUnlockOptions,
+  IIngestProgress,
+  IIngestComplete,
+  IVerifyError,
+  IRepositoryConfig,
+} from './interfaces.js';
+
+/**
+ * Content-addressed incremental backup engine.
+ *
+ * Provides deduplicated, optionally encrypted, gzip-compressed storage
+ * for arbitrary data streams with full snapshot history.
+ */
+export class ContainerArchive {
+  private bridge: plugins.smartrust.RustBridge<TContainerArchiveCommands>;
+  private repoPath: string;
+  private spawned = false;
+
+  // Event subjects
+  public ingestProgress = new plugins.smartrx.rxjs.Subject<IIngestProgress>();
+  public ingestComplete = new plugins.smartrx.rxjs.Subject<IIngestComplete>();
+  public verifyError = new plugins.smartrx.rxjs.Subject<IVerifyError>();
+
+  private constructor(repoPath: string) {
+    this.repoPath = plugins.path.resolve(repoPath);
+
+    const packageDir = plugins.path.resolve(
+      plugins.path.dirname(new URL(import.meta.url).pathname),
+      '..',
+    );
+
+    this.bridge = new plugins.smartrust.RustBridge<TContainerArchiveCommands>({
+      binaryName: 'containerarchive',
+      localPaths: [
+        plugins.path.join(packageDir, 'dist_rust', 'containerarchive'),
+      ],
+      readyTimeoutMs: 30000,
+      requestTimeoutMs: 300000,
+    });
+
+    // Listen for events from the Rust binary
+    this.bridge.on('event', (event: { event: string; data: any }) => {
+      if (event.event === 'progress') {
+        this.ingestProgress.next(event.data);
+      }
+    });
+  }
+
+  private async ensureSpawned(): Promise<void> {
+    if (!this.spawned) {
+      await this.bridge.spawn();
+      this.spawned = true;
+    }
+  }
+
+  /**
+   * Initialize a new repository at the given path.
+   */
+  static async init(repoPath: string, options?: IInitOptions): Promise<ContainerArchive> {
+    const instance = new ContainerArchive(repoPath);
+    await instance.ensureSpawned();
+
+    await instance.bridge.sendCommand('init', {
+      path: instance.repoPath,
+      passphrase: options?.passphrase,
+    });
+
+    return instance;
+  }
+
+  /**
+   * Open an existing repository at the given path.
+   */
+  static async open(repoPath: string, options?: IOpenOptions): Promise<ContainerArchive> {
+    const instance = new ContainerArchive(repoPath);
+    await instance.ensureSpawned();
+
+    await instance.bridge.sendCommand('open', {
+      path: instance.repoPath,
+      passphrase: options?.passphrase,
+    });
+
+    return instance;
+  }
+
+  /**
+   * Ingest a single data stream into the repository.
+   */
+  async ingest(
+    inputStream: NodeJS.ReadableStream,
+    options?: IIngestOptions,
+  ): Promise<ISnapshot> {
+    const socketPath = plugins.path.join(
+      plugins.os.tmpdir(),
+      `containerarchive-ingest-${Date.now()}-${Math.random().toString(36).slice(2)}.sock`,
+    );
+
+    // Create Unix socket server that Rust will connect to
+    const { promise: dataTransferred, server } = await this.createSocketServer(
+      socketPath,
+      inputStream,
+    );
+
+    try {
+      // Send ingest command to Rust (Rust connects to our socket)
+      const result = await this.bridge.sendCommand('ingest', {
+        socketPath,
+        tags: options?.tags,
+        items: options?.items || [{ name: 'data', type: 'data' }],
+      });
+
+      // Wait for data transfer to complete
+      await dataTransferred;
+
+      const snapshot = result.snapshot;
+      this.ingestComplete.next({
+        snapshotId: snapshot.id,
+        originalSize: snapshot.originalSize,
+        storedSize: snapshot.storedSize,
+        newChunks: snapshot.newChunks,
+        reusedChunks: snapshot.reusedChunks,
+      });
+
+      return snapshot;
+    } finally {
+      server.close();
+      // Clean up socket file
+      try {
+        plugins.fs.unlinkSync(socketPath);
+      } catch {}
+    }
+  }
+
+  /**
+   * Ingest multiple data streams as a single multi-item snapshot.
+   * Each item gets its own Unix socket for parallel data transfer.
+   */
+  async ingestMulti(
+    items: IIngestItem[],
+    options?: IIngestOptions,
+  ): Promise<ISnapshot> {
+    if (items.length === 0) {
+      throw new Error('At least one item is required');
+    }
+
+    // Create one socket per item
+    const sockets: Array<{
+      socketPath: string;
+      promise: Promise<void>;
+      server: plugins.net.Server;
+    }> = [];
+
+    const itemOptions: Array<{
+      name: string;
+      type: string;
+      socketPath: string;
+    }> = [];
+
+    try {
+      for (const item of items) {
+        const socketPath = plugins.path.join(
+          plugins.os.tmpdir(),
+          `containerarchive-ingest-${Date.now()}-${Math.random().toString(36).slice(2)}.sock`,
+        );
+
+        const { promise, server } = await this.createSocketServer(socketPath, item.stream);
+        sockets.push({ socketPath, promise, server });
+        itemOptions.push({
+          name: item.name,
+          type: item.type || 'data',
+          socketPath,
+        });
+      }
+
+      // Send ingestMulti command to Rust with per-item socket paths
+      const result = await this.bridge.sendCommand('ingestMulti', {
+        tags: options?.tags,
+        items: itemOptions,
+      });
+
+      // Wait for all data transfers
+      await Promise.all(sockets.map((s) => s.promise));
+
+      const snapshot = result.snapshot;
+      this.ingestComplete.next({
+        snapshotId: snapshot.id,
+        originalSize: snapshot.originalSize,
+        storedSize: snapshot.storedSize,
+        newChunks: snapshot.newChunks,
+        reusedChunks: snapshot.reusedChunks,
+      });
+
+      return snapshot;
+    } finally {
+      for (const s of sockets) {
+        s.server.close();
+        try { plugins.fs.unlinkSync(s.socketPath); } catch {}
+      }
+    }
+  }
+
+  /**
+   * List snapshots with optional filtering.
+   */
+  async listSnapshots(filter?: ISnapshotFilter): Promise<ISnapshot[]> {
+    const result = await this.bridge.sendCommand('listSnapshots', {
+      filter,
+    });
+    return result.snapshots;
+  }
+
+  /**
+   * Get details of a specific snapshot.
+   */
+  async getSnapshot(snapshotId: string): Promise<ISnapshot> {
+    const result = await this.bridge.sendCommand('getSnapshot', {
+      snapshotId,
+    });
+    return result.snapshot;
+  }
+
+  /**
+   * Restore a snapshot to a ReadableStream.
+   */
+  async restore(
+    snapshotId: string,
+    options?: IRestoreOptions,
+  ): Promise<NodeJS.ReadableStream> {
+    const socketPath = plugins.path.join(
+      plugins.os.tmpdir(),
+      `containerarchive-restore-${Date.now()}-${Math.random().toString(36).slice(2)}.sock`,
+    );
+
+    // Create Unix socket server that Rust will connect to and write data
+    const { readable, server } = await this.createRestoreSocketServer(socketPath);
+
+    // Send restore command to Rust (Rust connects and writes data)
+    // Don't await — let it run in parallel with reading
+    this.bridge.sendCommand('restore', {
+      snapshotId,
+      socketPath,
+      item: options?.item,
+    }).catch((err) => {
+      readable.destroy(err);
+    }).finally(() => {
+      server.close();
+      try {
+        plugins.fs.unlinkSync(socketPath);
+      } catch {}
+    });
+
+    return readable;
+  }
+
+  /**
+   * Verify repository integrity.
+   */
+  async verify(options?: IVerifyOptions): Promise<IVerifyResult> {
+    const result = await this.bridge.sendCommand('verify', {
+      level: options?.level || 'standard',
+    });
+
+    for (const error of result.errors) {
+      this.verifyError.next(error);
+    }
+
+    return result;
+  }
+
+  /**
+   * Repair repository (rebuild index, remove stale locks).
+   */
+  async repair(): Promise<IRepairResult> {
+    return this.bridge.sendCommand('repair', {});
+  }
+
+  /**
+   * Prune old snapshots and garbage-collect unreferenced packs.
+   */
+  async prune(retention: IRetentionPolicy, dryRun = false): Promise<IPruneResult> {
+    return this.bridge.sendCommand('prune', {
+      retention,
+      dryRun,
+    });
+  }
+
+  /**
+   * Rebuild the global index from pack .idx files.
+   */
+  async reindex(): Promise<void> {
+    await this.bridge.sendCommand('reindex', {});
+  }
+
+  /**
+   * Remove locks from the repository.
+   */
+  async unlock(options?: IUnlockOptions): Promise<void> {
+    await this.bridge.sendCommand('unlock', {
+      force: options?.force,
+    });
+  }
+
+  /**
+   * Subscribe to events.
+   */
+  on(event: 'ingest:progress', handler: (data: IIngestProgress) => void): plugins.smartrx.rxjs.Subscription;
+  on(event: 'ingest:complete', handler: (data: IIngestComplete) => void): plugins.smartrx.rxjs.Subscription;
+  on(event: 'verify:error', handler: (data: IVerifyError) => void): plugins.smartrx.rxjs.Subscription;
+  on(event: string, handler: (data: any) => void): plugins.smartrx.rxjs.Subscription {
+    switch (event) {
+      case 'ingest:progress':
+        return this.ingestProgress.subscribe(handler);
+      case 'ingest:complete':
+        return this.ingestComplete.subscribe(handler);
+      case 'verify:error':
+        return this.verifyError.subscribe(handler);
+      default:
+        throw new Error(`Unknown event: ${event}`);
+    }
+  }
+
+  /**
+   * Close the repository and terminate the Rust process.
+   */
+  async close(): Promise<void> {
+    try {
+      await this.bridge.sendCommand('close', {});
+    } catch {
+      // Ignore errors during close
+    }
+    this.bridge.kill();
+    this.spawned = false;
+
+    this.ingestProgress.complete();
+    this.ingestComplete.complete();
+    this.verifyError.complete();
+  }
+
+  // ==================== Private Helpers ====================
+
+  /**
+   * Create a Unix socket server that accepts a connection from Rust
+   * and pipes the inputStream to it (for ingest).
+   */
+  private createSocketServer(
+    socketPath: string,
+    inputStream: NodeJS.ReadableStream,
+  ): Promise<{
+    promise: Promise<void>;
+    server: plugins.net.Server;
+  }> {
+    return new Promise((resolve, reject) => {
+      const server = plugins.net.createServer((socket) => {
+        // Pipe input data to the Rust process via socket
+        const readableStream = inputStream as NodeJS.ReadableStream;
+        (readableStream as any).pipe(socket);
+      });
+
+      server.on('error', reject);
+
+      server.listen(socketPath, () => {
+        const promise = new Promise<void>((res) => {
+          server.on('close', () => res());
+          // Also resolve after a connection is handled
+          server.once('connection', (socket) => {
+            socket.on('end', () => {
+              res();
+            });
+            socket.on('error', () => {
+              res();
+            });
+          });
+        });
+        resolve({ promise, server });
+      });
+    });
+  }
+
+  /**
+   * Create a Unix socket server that accepts a connection from Rust
+   * and provides a ReadableStream of the received data (for restore).
+   */
+  private createRestoreSocketServer(
+    socketPath: string,
+  ): Promise<{
+    readable: plugins.stream.PassThrough;
+    server: plugins.net.Server;
+  }> {
+    return new Promise((resolve, reject) => {
+      const passthrough = new plugins.stream.PassThrough();
+      const server = plugins.net.createServer((socket) => {
+        socket.pipe(passthrough);
+      });
+
+      server.on('error', reject);
+
+      server.listen(socketPath, () => {
+        resolve({ readable: passthrough, server });
+      });
+    });
+  }
+}

package/ts/index.ts

@@ -0,0 +1,2 @@
+export * from './classes.containerarchive.js';
+export * from './interfaces.js';