cbyte 1.0.0

package/README.md

@@ -0,0 +1,208 @@
+# cbyte
+
+**cbyte** is a Node.js CLI utility for content-addressed, deduplicated storage using IPFS. It allows you to pack files into manifests (`.cbyte`), reconstruct them, verify integrity, and manage IPFS pinning.
+
+---
+
+## Features
+
+* Deduplicated storage using SHA-256 hashes
+* Fixed-size chunking for large files
+* IPFS raw block backend (no UnixFS overhead)
+* Deterministic `.cbyte` manifests
+* Parallel unpacking with safe ordered writes
+* ASCII progress bar during unpack
+* CLI commands: `pack`, `unpack`, `verify`, `pin`
+
+---
+
+## Requirements
+
+* Node.js >= 20
+* npm
+* IPFS (Kubo) >= 0.21
+* Linux environment (POSIX-compliant filesystem)
+
+---
+
+## Installation
+
+### Using npm
+
+```bash
+npm install -g cbyte
+```
+
+### Manual (from GitHub or tarball)
+
+```bash
+git clone https://github.com/yourusername/cbyte.git
+cd cbyte
+npm install
+chmod +x cbyte.js
+sudo ln -sf $(pwd)/cbyte.js /usr/local/bin/cbyte
+```
+
+Ensure IPFS daemon is running:
+
+```bash
+ipfs daemon &
+```
+
+---
+
+## CLI Usage
+
+```text
+cbyte <command> [options]
+```
+
+### Commands
+
+| Command           | Description                                 |
+| ----------------- | ------------------------------------------- |
+| pack <file>       | Pack a file into a `.cbyte` manifest        |
+| unpack <manifest> | Reconstruct the original file from manifest |
+| verify <manifest> | Verify availability & integrity of blocks   |
+| pin <manifest>    | Pin all blocks referenced by manifest       |
+
+### Options
+
+* `-o, --output <file>`: specify output file (manifest or restored file)
+* `--chunk-size <bytes>`: size of chunks in bytes (default 4096)
+* `--pin`: automatically pin blocks during pack
+* `-h, --help`: show help for command
+
+---
+
+## Examples
+
+```bash
+# Pack a file
+cbyte pack bigfile.bin -o bigfile.cbyte --chunk-size 4096 --pin
+
+# Unpack with ASCII progress
+cbyte unpack bigfile.cbyte -o bigfile_restored.bin
+
+# Verify blocks
+cbyte verify bigfile.cbyte
+
+# Pin all blocks
+cbyte pin bigfile.cbyte
+```
+
+ASCII progress bar example during unpack:
+
+```
+[======================        ] 62.5% (500/800)
+```
+
+---
+
+## Manifest Format
+
+```json
+{
+  "format": "cbyte",
+  "version": 1,
+  "chunk_size": 4096,
+  "original_size": 5242880,
+  "chunks": [
+    { "cid": "bafkreigh2akiscaildcg...", "length": 4096, "count": 2 }
+  ]
+}
+```
+
+* `cid`: IPFS content identifier
+* `length`: bytes in the chunk
+* `count`: repeated occurrences
+* `chunk_size`: chunk size used during packing
+
+---
+
+## Testing
+
+```bash
+# Small file
+echo "test data" > testfile.txt
+cbyte pack testfile.txt -o testfile.cbyte --chunk-size 32 --pin
+cbyte unpack testfile.cbyte -o restored.txt
+diff testfile.txt restored.txt
+
+# Large file
+dd if=/dev/urandom of=bigfile.bin bs=1M count=5
+cbyte pack bigfile.bin -o bigfile.cbyte --chunk-size 4096 --pin
+cbyte unpack bigfile.cbyte -o bigfile_restored.bin
+diff bigfile.bin bigfile_restored.bin
+```
+
+---
+
+## Safety Notes
+
+* Fetching or unpacking fails if any block is missing
+* Parallel fetching does not reorder writes; output is deterministic
+* CID ensures cryptographic integrity
+* Use `--pin` to persist blocks on your IPFS node
+
+---
+
+## License
+
+MIT
+
+---
+
+## Repository and Issues
+
+* GitHub: [https://github.com/yourusername/cbyte](https://github.com/yourusername/cbyte)
+* Issues: [https://github.com/yourusername/cbyte/issues](https://github.com/yourusername/cbyte/issues)
+
+
+**cbyte** is a Node.js CLI utility for content-addressed, deduplicated storage using IPFS. It allows you to pack files into manifests (`.cbyte`), reconstruct them, verify integrity, and manage IPFS pinning.
+
+## Features
+- Deduplicated storage using SHA-256 hashes
+- Fixed-size chunking for large files
+- IPFS raw block backend (no UnixFS overhead)
+- Deterministic `.cbyte` manifests
+- Parallel unpacking with safe ordered writes
+- ASCII progress bar during unpack
+- CLI commands: `pack`, `unpack`, `verify`, `pin`
+
+## Installation
+
+### Using npm
+```
+npm install -g cbyte
+```
+
+### Manual
+```
+git clone https://github.com/yourusername/cbyte.git
+cd cbyte
+npm install
+chmod +x cbyte.js
+sudo ln -sf $(pwd)/cbyte.js /usr/local/bin/cbyte
+```
+
+Ensure IPFS daemon is running:
+```
+ipfs daemon &
+```
+
+## CLI Usage
+```
+cbyte <command> [options]
+```
+
+Commands: `pack`, `unpack`, `verify`, `pin`
+
+## Examples
+```
+cbyte pack bigfile.bin -o bigfile.cbyte --chunk-size 4096 --pin
+cbyte unpack bigfile.cbyte -o bigfile_restored.bin
+cbyte verify bigfile.cbyte
+cbyte pin bigfile.cbyte
+```
+

package/cbyte.js

@@ -0,0 +1,230 @@
+#!/usr/bin/env node
+
+// Prevent MaxListenersExceededWarning
+require('events').defaultMaxListeners = 6000;
+
+const fs = require('fs');
+const crypto = require('crypto');
+const { program } = require('commander');
+const { request } = require('undici');
+const FormData = require('form-data');
+const os = require('os');
+const path = require('path');
+const { spawn } = require('child_process');
+const net = require('net');
+
+const IPFS_API = process.env.IPFS_API || 'http://127.0.0.1:5001/api/v0';
+
+// ------------------- IPFS Daemon Helpers -------------------
+function isPortOpen(host, port) {
+  return new Promise((resolve) => {
+    const socket = new net.Socket();
+    socket.setTimeout(1000);
+    socket.once('connect', () => { socket.destroy(); resolve(true); });
+    socket.once('timeout', () => { socket.destroy(); resolve(false); });
+    socket.once('error', () => { resolve(false); });
+    socket.connect(port, host);
+  });
+}
+
+async function ensureIpfsDaemon() {
+  const host = '127.0.0.1';
+  const port = 5001;
+
+  const running = await isPortOpen(host, port);
+  if (running) return;
+
+  console.log('IPFS daemon not running. Starting it now...');
+  const daemon = spawn('ipfs', ['daemon'], { stdio: 'ignore', detached: true });
+  daemon.unref();
+
+  let attempts = 0;
+  while (!(await isPortOpen(host, port))) {
+    if (attempts++ > 20) throw new Error('Failed to start IPFS daemon.');
+    await new Promise(r => setTimeout(r, 500));
+  }
+  console.log('IPFS daemon started.');
+}
+
+// ------------------- Utility Functions -------------------
+function sha256(data) {
+  return crypto.createHash('sha256').update(data).digest('hex');
+}
+
+function renderProgress(completed, total, label = '') {
+  const width = 40;
+  const ratio = completed / total;
+  const filled = Math.floor(ratio * width);
+  const bar = '='.repeat(filled) + ' '.repeat(width - filled);
+  process.stderr.write(`\r${label} [${bar}] ${(ratio*100).toFixed(1)}% (${completed}/${total})`);
+}
+
+// ------------------- IPFS Block Operations -------------------
+async function ipfsBlockPut(data) {
+  const form = new FormData();
+  form.append('file', data, { filename: 'chunk' });
+
+  const res = await request(`${IPFS_API}/block/put?format=raw&mhtype=sha2-256`, {
+    method: 'POST',
+    body: form,
+    headers: form.getHeaders()
+  });
+
+  const body = await res.body.text();
+  const json = JSON.parse(body);
+  return json.Key;
+}
+
+async function ipfsBlockGet(cid) {
+  const res = await request(`${IPFS_API}/block/get?arg=${cid}`);
+  const buffer = await res.body.arrayBuffer();
+  return Buffer.from(buffer);
+}
+
+// ------------------- Pack Function -------------------
+async function pack(filePath, output, chunkSize = 4096, pin = false) {
+  if (!output) {
+    const base = path.basename(filePath, path.extname(filePath));
+    output = `${base}.cbyte`;
+  }
+
+  const data = fs.readFileSync(filePath);
+  const chunksMap = {};
+  for (let offset = 0; offset < data.length; offset += chunkSize) {
+    const chunk = data.slice(offset, offset + chunkSize);
+    const hash = sha256(chunk);
+    if (!chunksMap[hash]) chunksMap[hash] = { chunk, count: 0 };
+    chunksMap[hash].count++;
+  }
+
+  const manifest = {
+    format: 'cbyte',
+    version: 1,
+    chunk_size: chunkSize,
+    original_size: data.length,
+    original_ext: path.extname(filePath), // store original file extension
+    chunks: []
+  };
+
+  const hashes = Object.keys(chunksMap);
+  const total = hashes.length;
+  let uploaded = 0;
+
+  const concurrency = Math.max(1, Math.min(8, os.cpus().length));
+  for (let i = 0; i < hashes.length; i += concurrency) {
+    const batch = hashes.slice(i, i + concurrency);
+    await Promise.all(batch.map(async (hash) => {
+      const info = chunksMap[hash];
+      const cid = await ipfsBlockPut(info.chunk);
+      if (pin) await request(`${IPFS_API}/pin/add?arg=${cid}`);
+      manifest.chunks.push({ cid, length: info.chunk.length, count: info.count });
+      uploaded++;
+      renderProgress(uploaded, total, 'Uploading chunks');
+    }));
+  }
+  process.stderr.write('\n');
+
+  fs.writeFileSync(output, JSON.stringify(manifest, null, 2));
+  console.log(`Packed ${filePath} -> ${output}`);
+}
+
+// ------------------- Unpack Function -------------------
+async function unpack(manifestPath, output) {
+  const manifest = JSON.parse(fs.readFileSync(manifestPath));
+  if (!output) {
+    const base = path.basename(manifestPath, path.extname(manifestPath));
+    const ext = manifest.original_ext || '.out';
+    output = `${base}${ext}`;
+  }
+
+  const outFd = fs.openSync(output, 'w');
+  const totalChunks = manifest.chunks.reduce((a, c) => a + c.count, 0);
+  let completed = 0;
+
+  const concurrency = Math.max(1, Math.min(8, os.cpus().length));
+  for (let i = 0; i < manifest.chunks.length; i += concurrency) {
+    const batch = manifest.chunks.slice(i, i + concurrency);
+    await Promise.all(batch.map(async (entry) => {
+      const data = await ipfsBlockGet(entry.cid);
+      for (let j = 0; j < entry.count; j++) {
+        fs.writeSync(outFd, data.slice(0, entry.length));
+        completed++;
+        renderProgress(completed, totalChunks, 'Unpacking');
+      }
+    }));
+  }
+
+  process.stderr.write('\n');
+  fs.ftruncateSync(outFd, manifest.original_size);
+  fs.closeSync(outFd);
+  console.log(`Unpacked -> ${output}`);
+}
+
+// ------------------- Verify Function -------------------
+async function verify(manifestPath) {
+  const manifest = JSON.parse(fs.readFileSync(manifestPath));
+  let verified = 0;
+  for (const { cid } of manifest.chunks) {
+    try {
+      await ipfsBlockGet(cid);
+      verified++;
+      renderProgress(verified, manifest.chunks.length, 'Verifying');
+    } catch (e) {
+      console.error(`Missing block ${cid}`);
+      process.exit(1);
+    }
+  }
+  process.stderr.write('\n');
+  console.log('All blocks verified');
+}
+
+// ------------------- Pin Function -------------------
+async function pin(manifestPath) {
+  const manifest = JSON.parse(fs.readFileSync(manifestPath));
+  let pinned = 0;
+  for (const { cid } of manifest.chunks) {
+    await request(`${IPFS_API}/pin/add?arg=${cid}`);
+    pinned++;
+    renderProgress(pinned, manifest.chunks.length, 'Pinning');
+  }
+  process.stderr.write('\n');
+  console.log('All blocks pinned');
+}
+
+// ------------------- CLI -------------------
+program.version('1.0.0');
+
+program
+  .command('pack <file>')
+  .option('-o, --output <file>', 'Output manifest')
+  .option('--chunk-size <bytes>', 'Chunk size', '4096')
+  .option('--pin', 'Pin blocks after upload', false)
+  .action(async (file, options) => {
+    await ensureIpfsDaemon();
+    await pack(file, options.output, parseInt(options['chunkSize']), options.pin);
+  });
+
+program
+  .command('unpack <manifest>')
+  .option('-o, --output <file>', 'Output file')
+  .action(async (manifest, options) => {
+    await ensureIpfsDaemon();
+    await unpack(manifest, options.output);
+  });
+
+program
+  .command('verify <manifest>')
+  .action(async (manifest) => {
+    await ensureIpfsDaemon();
+    await verify(manifest);
+  });
+
+program
+  .command('pin <manifest>')
+  .action(async (manifest) => {
+    await ensureIpfsDaemon();
+    await pin(manifest);
+  });
+
+program.parse(process.argv);
+

package/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "cbyte",
+  "version": "1.0.0",
+  "description": "Content-addressed deduplicated storage CLI using IPFS with automatic unpack extension retention",
+  "main": "cbyte.js",
+  "bin": { "cbyte": "./cbyte.js" },
+  "scripts": { "test": "node test.js" },
+  "keywords": ["ipfs","cli","deduplication","storage","content-addressed"],
+  "author": "Your Name <you@example.com>",
+  "license": "MIT",
+  "dependencies": {
+    "commander": "^11.0.0",
+    "undici": "^6.0.0",
+    "form-data": "^4.0.0"
+  },
+  "engines": { "node": ">=20" }
+}
+