cbyte 1.11.0 → 1.13.0

package/99367107.cbyte

@@ -1,6 +1,6 @@
 {
   "format": "cbyte",
-  "version": 1.1,
+  "version": 1.11,
   "chunk_size": 4096,
   "original_size": 16552,
   "original_ext": ".pdf",

package/README.md

@@ -1,208 +1,16 @@
-# cbyte
+# cbyte v1.13
 
-**cbyte** is a Node.js CLI utility for content-addressed, deduplicated storage using IPFS. It allows you to pack files into manifests (`.cbyte`), reconstruct them, verify integrity, and manage IPFS pinning.
-
----
+cbyte is an offline, content-addressed, deduplicated file packer built on Helia.
 
 ## Features
+- Streaming chunking (large-file safe)
+- Deterministic manifests
+- Persistent deduplicated blockstore
+- No IPFS daemon, no networking
+- Clean CLI exit
 
-* Deduplicated storage using SHA-256 hashes
-* Fixed-size chunking for large files
-* IPFS raw block backend (no UnixFS overhead)
-* Deterministic `.cbyte` manifests
-* Parallel unpacking with safe ordered writes
-* ASCII progress bar during unpack
-* CLI commands: `pack`, `unpack`, `verify`, `pin`
-
----
-
-## Requirements
-
-* Node.js >= 20
-* npm
-* IPFS (Kubo) >= 0.21
-* Linux environment (POSIX-compliant filesystem)
-
----
-
-## Installation
-
-### Using npm
-
-```bash
-npm install -g cbyte
-```
-
-### Manual (from GitHub or tarball)
-
-```bash
-git clone https://github.com/yourusername/cbyte.git
-cd cbyte
-npm install
-chmod +x cbyte.js
-sudo ln -sf $(pwd)/cbyte.js /usr/local/bin/cbyte
-```
-
-Ensure IPFS daemon is running:
-
-```bash
-ipfs daemon &
-```
-
----
-
-## CLI Usage
-
-```text
-cbyte <command> [options]
-```
-
-### Commands
-
-| Command           | Description                                 |
-| ----------------- | ------------------------------------------- |
-| pack <file>       | Pack a file into a `.cbyte` manifest        |
-| unpack <manifest> | Reconstruct the original file from manifest |
-| verify <manifest> | Verify availability & integrity of blocks   |
-| pin <manifest>    | Pin all blocks referenced by manifest       |
-
-### Options
-
-* `-o, --output <file>`: specify output file (manifest or restored file)
-* `--chunk-size <bytes>`: size of chunks in bytes (default 4096)
-* `--pin`: automatically pin blocks during pack
-* `-h, --help`: show help for command
-
----
-
-## Examples
-
-```bash
-# Pack a file
-cbyte pack bigfile.bin -o bigfile.cbyte --chunk-size 4096 --pin
-
-# Unpack with ASCII progress
-cbyte unpack bigfile.cbyte -o bigfile_restored.bin
-
-# Verify blocks
-cbyte verify bigfile.cbyte
-
-# Pin all blocks
-cbyte pin bigfile.cbyte
-```
-
-ASCII progress bar example during unpack:
-
-```
-[======================        ] 62.5% (500/800)
-```
-
----
-
-## Manifest Format
-
-```json
-{
-  "format": "cbyte",
-  "version": 1,
-  "chunk_size": 4096,
-  "original_size": 5242880,
-  "chunks": [
-    { "cid": "bafkreigh2akiscaildcg...", "length": 4096, "count": 2 }
-  ]
-}
-```
-
-* `cid`: IPFS content identifier
-* `length`: bytes in the chunk
-* `count`: repeated occurrences
-* `chunk_size`: chunk size used during packing
-
----
-
-## Testing
-
-```bash
-# Small file
-echo "test data" > testfile.txt
-cbyte pack testfile.txt -o testfile.cbyte --chunk-size 32 --pin
-cbyte unpack testfile.cbyte -o restored.txt
-diff testfile.txt restored.txt
-
-# Large file
-dd if=/dev/urandom of=bigfile.bin bs=1M count=5
-cbyte pack bigfile.bin -o bigfile.cbyte --chunk-size 4096 --pin
-cbyte unpack bigfile.cbyte -o bigfile_restored.bin
-diff bigfile.bin bigfile_restored.bin
-```
-
----
-
-## Safety Notes
-
-* Fetching or unpacking fails if any block is missing
-* Parallel fetching does not reorder writes; output is deterministic
-* CID ensures cryptographic integrity
-* Use `--pin` to persist blocks on your IPFS node
-
----
-
-## License
-
-MIT
-
----
-
-## Repository and Issues
-
-* GitHub: [https://github.com/yourusername/cbyte](https://github.com/yourusername/cbyte)
-* Issues: [https://github.com/yourusername/cbyte/issues](https://github.com/yourusername/cbyte/issues)
-
-
-**cbyte** is a Node.js CLI utility for content-addressed, deduplicated storage using IPFS. It allows you to pack files into manifests (`.cbyte`), reconstruct them, verify integrity, and manage IPFS pinning.
-
-## Features
-- Deduplicated storage using SHA-256 hashes
-- Fixed-size chunking for large files
-- IPFS raw block backend (no UnixFS overhead)
-- Deterministic `.cbyte` manifests
-- Parallel unpacking with safe ordered writes
-- ASCII progress bar during unpack
-- CLI commands: `pack`, `unpack`, `verify`, `pin`
-
-## Installation
-
-### Using npm
-```
-npm install -g cbyte
-```
-
-### Manual
-```
-git clone https://github.com/yourusername/cbyte.git
-cd cbyte
-npm install
-chmod +x cbyte.js
-sudo ln -sf $(pwd)/cbyte.js /usr/local/bin/cbyte
-```
-
-Ensure IPFS daemon is running:
-```
-ipfs daemon &
-```
-
-## CLI Usage
-```
-cbyte <command> [options]
-```
-
-Commands: `pack`, `unpack`, `verify`, `pin`
-
-## Examples
-```
-cbyte pack bigfile.bin -o bigfile.cbyte --chunk-size 4096 --pin
-cbyte unpack bigfile.cbyte -o bigfile_restored.bin
-cbyte verify bigfile.cbyte
-cbyte pin bigfile.cbyte
-```
+## Usage
+cbyte pack file.bin
+cbyte unpack file.bin.cbyte
 
+Requires Node.js >= 20.8.0

package/cbyte.js

@@ -1,23 +1,11 @@
 #!/usr/bin/env node
 
-// Increase listener limit to avoid warnings
-import evt from 'events';
-import fs from 'fs';
-import crypto from 'crypto';
-import { Command } from 'commander';
-import { createHelia } from 'helia';
-import * as Block from 'multiformats/block';
-import * as rawCodec from 'multiformats/codecs/raw';
-import * as sha256 from 'multiformats/hashes/sha2';
-import os from 'os';
-import path from 'path';
-
-evt.defaultMaxListeners = 6000;
-
-const program = new Command();
-let helia = null;
-
-// ---- Promise.withResolvers polyfill (Node < 22) ----
+const [NODE_MAJOR, NODE_MINOR] = process.versions.node.split('.').map(Number)
+if (NODE_MAJOR < 20 || (NODE_MAJOR === 20 && NODE_MINOR < 8)) {
+  console.error('cbyte requires Node.js >= 20.8.0')
+  process.exit(1)
+}
+
 if (typeof Promise.withResolvers !== 'function') {
   Promise.withResolvers = function () {
     let resolve, reject
@@ -29,174 +17,115 @@ if (typeof Promise.withResolvers !== 'function') {
   }
 }
 
-// Initialize Helia node once
-async function ensureHelia() {
-  if (!helia) {
-    helia = await createHelia();
-  }
-}
+const fs = require('fs')
+const path = require('path')
+const os = require('os')
+const crypto = require('crypto')
+const { program } = require('commander')
+const { EventEmitter } = require('events')
+
+EventEmitter.defaultMaxListeners = 6000
+
+async function createHeliaNode(storePath) {
+  const { createHelia } = await import('helia')
+  const { FsBlockstore } = await import('blockstore-fs')
 
-// Simple SHA‑256 progress bar
-function renderProgress(completed, total, label = '') {
-  const width = 40;
-  const ratio = completed / total;
-  const filled = Math.floor(ratio * width);
-  const bar = '='.repeat(filled) + ' '.repeat(width - filled);
-  process.stderr.write(`\r${label} [${bar}] ${(ratio * 100).toFixed(1)}% (${completed}/${total})`);
+  const blockstore = new FsBlockstore(storePath)
+  const helia = await createHelia({ blockstore, libp2p: false })
+  return { helia, blockstore }
 }
 
-// Pack operation with Helia
-async function pack(filePath, outFile, chunkSize = 4096, pin = false) {
-  await ensureHelia();
+function sha256(buf) {
+  return crypto.createHash('sha256').update(buf).digest('hex')
+}
 
-  if (!outFile) {
-    const base = path.basename(filePath, path.extname(filePath));
-    outFile = `${base}.cbyte`;
-  }
+function renderProgress(label, done, total) {
+  const width = 40
+  const ratio = total === 0 ? 1 : done / total
+  const filled = Math.floor(ratio * width)
+  const bar = '='.repeat(filled) + ' '.repeat(width - filled)
+  process.stderr.write(`\r${label} [${bar}] ${(ratio * 100).toFixed(1)}%`)
+}
 
-  const data = fs.readFileSync(filePath);
-  const chunksMap = {};
+async function pack(inputFile, outputFile, chunkSize) {
+  const storeDir = path.join(os.homedir(), '.cbyte', 'blocks')
+  fs.mkdirSync(storeDir, { recursive: true })
 
-  for (let offset = 0; offset < data.length; offset += chunkSize) {
-    const chunk = data.slice(offset, offset + chunkSize);
-    const hashVal = crypto.createHash('sha256').update(chunk).digest();
-    const hash = Buffer.from(hashVal).toString('hex');
-    if (!chunksMap[hash]) chunksMap[hash] = { chunk, count: 0 };
-    chunksMap[hash].count++;
-  }
+  const { helia, blockstore } = await createHeliaNode(storeDir)
+  const stat = fs.statSync(inputFile)
 
   const manifest = {
     format: 'cbyte',
-    version: 1.11,
+    version: '1.13',
+    original_size: stat.size,
     chunk_size: chunkSize,
-    original_size: data.length,
-    original_ext: path.extname(filePath),
+    extension: path.extname(inputFile),
     chunks: []
-  };
-
-  const hashes = Object.keys(chunksMap);
-  let uploaded = 0;
-
-  const concurrency = Math.max(1, Math.min(8, os.cpus().length));
-  for (let i = 0; i < hashes.length; i += concurrency) {
-    const batch = hashes.slice(i, i + concurrency);
-    await Promise.all(batch.map(async (h) => {
-      const info = chunksMap[h];
-
-      // Create a raw block; multiformats block API
-      const block = await Block.encode({
-        value: info.chunk,
-        codec: rawCodec,
-        hasher: sha256.sha256
-      });
-
-      await helia.blockstore.put(block.cid, block.bytes);
-
-      manifest.chunks.push({
-        cid: block.cid.toString(),
-        length: info.chunk.length,
-        count: info.count
-      });
-
-      uploaded++;
-      renderProgress(uploaded, hashes.length, 'Packing ...');
-    }));
   }
-  process.stderr.write('\n');
 
-  fs.writeFileSync(outFile, JSON.stringify(manifest, null, 2));
-  console.log(`Packed ${filePath} -> ${outFile}`);
-}
+  const seen = new Map()
+  let processed = 0
+
+  const stream = fs.createReadStream(inputFile, { highWaterMark: chunkSize })
+  for await (const chunk of stream) {
+    const hash = sha256(chunk)
+    let entry = seen.get(hash)
+    if (!entry) {
+      const cid = await blockstore.put(chunk)
+      entry = { cid: cid.toString(), length: chunk.length, count: 0 }
+      seen.set(hash, entry)
+      manifest.chunks.push(entry)
+    }
+    entry.count++
+    processed += chunk.length
+    renderProgress('Packing', processed, stat.size)
+  }
 
-// Unpack using Helia
-async function unpack(manifestPath, outFile) {
-  await ensureHelia();
+  process.stderr.write('\n')
+  manifest.manifest_hash = sha256(Buffer.from(JSON.stringify(manifest)))
+  fs.writeFileSync(outputFile, JSON.stringify(manifest, null, 2))
+  console.log(`Packed -> ${outputFile}`)
+  await helia.stop()
+  process.exit(0)
+}
 
-  const manifest = JSON.parse(fs.readFileSync(manifestPath));
-  if (!outFile) {
-    const base = path.basename(manifestPath, path.extname(manifestPath));
-    const ext = manifest.original_ext || '.bin';
-    outFile = `${base}${ext}`;
-  }
+async function unpack(manifestPath, outputFile) {
+  const manifest = JSON.parse(fs.readFileSync(manifestPath))
+  if (!outputFile) outputFile = `output${manifest.extension || ''}`
 
-  const fd = fs.openSync(outFile, 'w');
-  const total = manifest.chunks.reduce((sum, e) => sum + e.count, 0);
-  let completed = 0;
-
-  const concurrency = Math.max(1, Math.min(8, os.cpus().length));
-  for (let i = 0; i < manifest.chunks.length; i += concurrency) {
-    const batch = manifest.chunks.slice(i, i + concurrency);
-    await Promise.all(batch.map(async (entry) => {
-      const blockBytes = await helia.blockstore.get(entry.cid);
-      for (let j = 0; j < entry.count; j++) {
-        fs.writeSync(fd, blockBytes.slice(0, entry.length));
-        completed++;
-        renderProgress(completed, total, 'Unpacking... ');
-      }
-    }));
-  }
-  process.stderr.write('\n');
+  const storeDir = path.join(os.homedir(), '.cbyte', 'blocks')
+  const { helia, blockstore } = await createHeliaNode(storeDir)
 
-  fs.ftruncateSync(fd, manifest.original_size);
-  fs.closeSync(fd);
-  console.log(`Unpacked -> ${outFile}`);
-}
+  const fd = fs.openSync(outputFile, 'w')
+  let written = 0
 
-// Verify blocks exist in Helia
-async function verify(manifestPath) {
-  await ensureHelia();
-  const manifest = JSON.parse(fs.readFileSync(manifestPath));
-  let checked = 0;
-  for (const { cid } of manifest.chunks) {
-    try {
-      await helia.blockstore.get(cid);
-      checked++;
-      renderProgress(checked, manifest.chunks.length, 'Verifying');
-    } catch {
-      console.error(`Missing block ${cid}`);
-      process.exit(1);
+  for (const entry of manifest.chunks) {
+    const data = await blockstore.get(entry.cid)
+    for (let i = 0; i < entry.count; i++) {
+      fs.writeSync(fd, data.slice(0, entry.length))
+      written += entry.length
+      renderProgress('Unpacking', written, manifest.original_size)
     }
   }
-  process.stderr.write('\n');
-  console.log('All blocks verified');
-}
 
-// Pin via Helia’s pin API
-async function pin(manifestPath) {
-  await ensureHelia();
-  const manifest = JSON.parse(fs.readFileSync(manifestPath));
-  let pinned = 0;
-  for (const { cid } of manifest.chunks) {
-    await helia.pin.add(cid);
-    pinned++;
-    renderProgress(pinned, manifest.chunks.length, 'Pinning...');
-  }
-  process.stderr.write('\n');
-  console.log('All blocks pinned');
+  fs.ftruncateSync(fd, manifest.original_size)
+  fs.closeSync(fd)
+  process.stderr.write('\n')
+  console.log(`Unpacked -> ${outputFile}`)
+  await helia.stop()
+  process.exit(0)
 }
 
-// CLI definition
-program.version('1.10.0');
-
-program
-  .command('pack <file>')
-  .option('-o, --output <file>', 'Output manifest')
-  .option('--chunk-size <bytes>', 'Chunk size', '4096')
-  .option('--pin', 'Pin blocks after pack', false)
-  .action((file, opts) => pack(file, opts.output, parseInt(opts['chunkSize']), opts.pin));
-
-program
-  .command('unpack <manifest>')
-  .option('-o, --output <file>', 'Output file')
-  .action((m, opts) => unpack(m, opts.output));
-
-program
-  .command('verify <manifest>')
-  .action((m) => verify(m));
-
-program
-  .command('pin <manifest>')
-  .action((m) => pin(m));
+program.version('1.13')
+program.command('pack <file>')
+  .option('-o, --output <file>')
+  .option('-c, --chunk-size <bytes>', 'Chunk size', '4096')
+  .action((file, opts) => {
+    pack(file, opts.output || path.basename(file) + '.cbyte', parseInt(opts.chunkSize))
+  })
 
-program.parse(process.argv);
+program.command('unpack <manifest>')
+  .option('-o, --output <file>')
+  .action((m, opts) => unpack(m, opts.output))
 
+program.parse(process.argv)

package/package.json

@@ -1,33 +1,14 @@
 {
   "name": "cbyte",
-  "type": "module",
-  "version": "1.11.0",
-  "description": "Content-addressed deduplicated storage CLI using IPFS with automatic unpack extension retention",
-  "main": "cbyte.js",
-  "bin": {
-    "cbyte": "cbyte.js"
-  },
-  "scripts": {
-    "test": "node test.js"
-  },
-  "keywords": [
-    "ipfs",
-    "cli",
-    "deduplication",
-    "storage",
-    "content-addressed"
-  ],
-  "author": "Coperbyte <coperbyte@coperbyte.com>",
-  "license": "MIT",
+  "version": "1.13.0",
+  "description": "Content-addressed deduplicated packer using embedded Helia (offline)",
+  "bin": { "cbyte": "./cbyte.js" },
+  "type": "commonjs",
+  "engines": { "node": ">=20.8.0" },
   "dependencies": {
-    "@helia/unixfs": "^7.0.1",
     "commander": "^11.0.0",
-    "form-data": "^4.0.0",
-    "helia": "^6.0.16",
-    "multiformats": "^13.4.2",
-    "undici": "^6.0.0"
+    "helia": "^4.0.0",
+    "blockstore-fs": "^2.0.0"
   },
-  "engines": {
-    "node": ">=20.8.0"
-  }
+  "license": "MIT"
 }