@enbox/dwn-sql-store 0.0.7 → 0.0.8

package/dist/esm/src/blockstore-sql.js

@@ -0,0 +1,117 @@
+import { CID } from 'multiformats';
+/**
+ * SQL-backed implementation of the `Blockstore` v5 interface, scoped to a
+ * single `rootDataCid`. All block operations are constrained to the blocks
+ * belonging to this root CID in the `dataBlocks` table.
+ *
+ * Used by `ipfs-unixfs-importer` (during `put()`) and `ipfs-unixfs-exporter`
+ * (during `get()`) to store and retrieve individual DAG-PB blocks.
+ *
+ * The Kysely instance and database connection are managed externally by
+ * `DataStoreSql`. This class does not own the connection lifecycle.
+ */
+export class BlockstoreSql {
+    #db;
+    #rootDataCid;
+    constructor(db, rootDataCid) {
+        this.#db = db;
+        this.#rootDataCid = rootDataCid;
+    }
+    async open() {
+        // No-op: connection managed by DataStoreSql.
+    }
+    async close() {
+        // No-op: connection managed by DataStoreSql.
+    }
+    async put(key, val, _options) {
+        const blockCid = key.toString();
+        await this.#db
+            .insertInto('dataBlocks')
+            .values({
+            rootDataCid: this.#rootDataCid,
+            blockCid,
+            data: Buffer.from(val),
+        })
+            .execute();
+        return key;
+    }
+    async get(key, _options) {
+        const result = await this.#db
+            .selectFrom('dataBlocks')
+            .select('data')
+            .where('rootDataCid', '=', this.#rootDataCid)
+            .where('blockCid', '=', key.toString())
+            .executeTakeFirst();
+        if (!result) {
+            throw new Error(`BlockstoreSql: block not found for rootDataCid=${this.#rootDataCid}, blockCid=${key}`);
+        }
+        return new Uint8Array(result.data);
+    }
+    async has(key, _options) {
+        const result = await this.#db
+            .selectFrom('dataBlocks')
+            .select('blockCid')
+            .where('rootDataCid', '=', this.#rootDataCid)
+            .where('blockCid', '=', key.toString())
+            .executeTakeFirst();
+        return result !== undefined;
+    }
+    async delete(key, _options) {
+        await this.#db
+            .deleteFrom('dataBlocks')
+            .where('rootDataCid', '=', this.#rootDataCid)
+            .where('blockCid', '=', key.toString())
+            .execute();
+    }
+    async isEmpty(_options) {
+        const result = await this.#db
+            .selectFrom('dataBlocks')
+            .select('blockCid')
+            .where('rootDataCid', '=', this.#rootDataCid)
+            .executeTakeFirst();
+        return result === undefined;
+    }
+    async *putMany(source, options) {
+        for await (const entry of source) {
+            await this.put(entry.cid, entry.block, options);
+            yield entry.cid;
+        }
+    }
+    async *getMany(source, options) {
+        for await (const key of source) {
+            yield {
+                cid: key,
+                block: await this.get(key, options),
+            };
+        }
+    }
+    async *getAll(_options) {
+        const rows = await this.#db
+            .selectFrom('dataBlocks')
+            .select(['blockCid', 'data'])
+            .where('rootDataCid', '=', this.#rootDataCid)
+            .execute();
+        for (const row of rows) {
+            yield {
+                cid: CID.parse(row.blockCid),
+                block: new Uint8Array(row.data),
+            };
+        }
+    }
+    async *deleteMany(source, options) {
+        for await (const key of source) {
+            await this.delete(key, options);
+            yield key;
+        }
+    }
+    /**
+     * Deletes all blocks for this rootDataCid.
+     */
+    async clear() {
+        await this.#db
+            .deleteFrom('dataBlocks')
+            .where('rootDataCid', '=', this.#rootDataCid)
+            .execute();
+    }
+}
+//# sourceMappingURL=blockstore-sql.js.map

package/dist/esm/src/blockstore-sql.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"blockstore-sql.js","sourceRoot":"","sources":["../../../src/blockstore-sql.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,GAAG,EAAE,MAAM,cAAc,CAAC;AAEnC;;;;;;;;;;GAUG;AACH,MAAM,OAAO,aAAa;IACxB,GAAG,CAA0B;IAC7B,YAAY,CAAS;IAErB,YAAY,EAA2B,EAAE,WAAmB;QAC1D,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC;QACd,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;IAClC,CAAC;IAEM,KAAK,CAAC,IAAI;QACf,6CAA6C;IAC/C,CAAC;IAEM,KAAK,CAAC,KAAK;QAChB,6CAA6C;IAC/C,CAAC;IAEM,KAAK,CAAC,GAAG,CAAC,GAAQ,EAAE,GAAe,EAAE,QAAuB;QACjE,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC;QAEhC,MAAM,IAAI,CAAC,GAAG;aACX,UAAU,CAAC,YAAY,CAAC;aACxB,MAAM,CAAC;YACN,WAAW,EAAG,IAAI,CAAC,YAAY;YAC/B,QAAQ;YACR,IAAI,EAAU,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC;SAC/B,CAAC;aACD,OAAO,EAAE,CAAC;QAEb,OAAO,GAAG,CAAC;IACb,CAAC;IAEM,KAAK,CAAC,GAAG,CAAC,GAAQ,EAAE,QAAuB;QAChD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG;aAC1B,UAAU,CAAC,YAAY,CAAC;aACxB,MAAM,CAAC,MAAM,CAAC;aACd,KAAK,CAAC,aAAa,EAAE,GAAG,EAAE,IAAI,CAAC,YAAY,CAAC;aAC5C,KAAK,CAAC,UAAU,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC;aACtC,gBAAgB,EAAE,CAAC;QAEtB,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,kDAAkD,IAAI,CAAC,YAAY,cAAc,GAAG,EAAE,CAAC,CAAC;QAC1G,CAAC;QAED,OAAO,IAAI,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACrC,CAAC;IAEM,KAAK,CAAC,GAAG,CAAC,GAAQ,EAAE,QAAuB;QAChD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG;aAC1B,UAAU,CAAC,YAAY,CAAC;aACxB,MAAM,CAAC,UAAU,CAAC;aAClB,KAAK,CAAC,aAAa,EAAE,GAAG,EAAE,IAAI,CAAC,YAAY,CAAC;aAC5C,KAAK,CAAC,UAAU,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC;aACtC,gBAAgB,EAAE,CAAC;QAEtB,OAAO,MAAM,KAAK,SAAS,CAAC;IAC9B,CAAC;IAEM,KAAK,CAAC,MAAM,CAAC,GAAQ,EAAE,QAAuB;QACnD,MAAM,IAAI,CAAC,GAAG;aACX,UAAU,CAAC,YAAY,CAAC;aACxB,KAAK,CAAC,aAAa,EAAE,GAAG,EAAE,IAAI,CAAC,YAAY,CAAC;aAC5C,KAAK,CAAC,UAAU,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC;aACtC,OAAO,EAAE,CAAC;IACf,CAAC;IAEM,KAAK,CAAC,OAAO,CAAC,QAAuB;QAC1C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG;aAC1B,UAAU,CAAC,YAAY,CAAC;aACxB,MAAM,CAAC,UAAU,CAAC;aAClB,KAAK,CAAC,aAAa,EAAE,GAAG,EAAE,IAAI,CAAC,YAAY,CAAC;aAC5C,gBAAgB,EAAE,CAAC;QAEtB,OAAO,MAAM,KAAK,SAAS,CAAC;IAC9B,CAAC;IAEM,KAAK,CAAC,CAAE,OAAO,CAAC,MAA2B,EAAE,OAAsB;QACxE,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YAChD,MAAM,KAAK,CAAC,GAAG,CAAC;QAClB,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,CAAE,OAAO,CAAC,MAA0B,EAAE,OAAsB;QACvE,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;YAC/B,MAAM;gBACJ,GAAG,EAAK,GAAG;gBACX,KAAK,EAAG,MAAM,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC;aACrC,CAAC;QACJ,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,CAAE,MAAM,CAAC,QAAuB;QAC3C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,GAAG;aACxB,UAAU,CAAC,YAAY,CAAC;aACxB,MAAM,CAAC,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;aAC5B,KAAK,CAAC,aAAa,EAAE,GAAG,EAAE,IAAI,CAAC,YAAY,CAAC;aAC5C,OAAO,EAAE,CAAC;QAEb,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM;gBACJ,GAAG,EAAK,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC;gBAC/B,KAAK,EAAG,IAAI,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC;aACjC,CAAC;QACJ,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,CAAE,UAAU,CAAC,MAA0B,EAAE,OAAsB;QAC1E,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;YAC/B,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YAChC,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,KAAK;QAChB,MAAM,IAAI,CAAC,GAAG;aACX,UAAU,CAAC,YAAY,CAAC;aACxB,KAAK,CAAC,aAAa,EAAE,GAAG,EAAE,IAAI,CAAC,YAAY,CAAC;aAC5C,OAAO,EAAE,CAAC;IACf,CAAC;CACF"}

package/dist/esm/src/data-store-s3.js

@@ -0,0 +1,243 @@
+import { DataStream } from '@enbox/dwn-sdk-js';
+import { Kysely } from 'kysely';
+import { Readable } from 'stream';
+import { Upload } from '@aws-sdk/lib-storage';
+import { DeleteObjectCommand, DeleteObjectsCommand, GetObjectCommand, ListObjectsV2Command, PutObjectCommand, S3Client, } from '@aws-sdk/client-s3';
+/**
+ * S3-backed implementation of {@link DataStore} with SQL-based reference
+ * tracking for content-addressed deduplication.
+ *
+ * Data is stored as whole S3 objects keyed by `dataCid`. The same `dataCid`
+ * maps to a single S3 object regardless of how many (tenant, recordId) pairs
+ * reference it. A `dataRefs` SQL table tracks references; blocks are
+ * garbage-collected from S3 when the last ref is deleted.
+ *
+ * For files over `partSize` (default 5MB), the AWS SDK Upload helper
+ * automatically uses multipart upload with bounded memory
+ * (`queueSize * partSize`).
+ */
+export class DataStoreS3 {
+    #dialect;
+    #db = null;
+    #s3;
+    #bucket;
+    #partSize;
+    #queueSize;
+    constructor(config) {
+        this.#dialect = config.dialect;
+        this.#bucket = config.bucket;
+        this.#partSize = config.partSize ?? 5 * 1024 * 1024; // 5 MB
+        this.#queueSize = config.queueSize ?? 4;
+        this.#s3 = config.s3Client ?? new S3Client({
+            region: config.region ?? 'us-east-1',
+            endpoint: config.endpoint,
+            forcePathStyle: config.forcePathStyle ?? false,
+            credentials: config.credentials,
+        });
+    }
+    async open() {
+        if (this.#db) {
+            return;
+        }
+        this.#db = new Kysely({ dialect: this.#dialect });
+        await this.#ensureRefsTable();
+    }
+    async close() {
+        await this.#db?.destroy();
+        this.#db = null;
+    }
+    async get(tenant, recordId, dataCid) {
+        const db = this.#getDb('get');
+        const ref = await db
+            .selectFrom('dataRefs')
+            .select('dataSize')
+            .where('tenant', '=', tenant)
+            .where('recordId', '=', recordId)
+            .where('dataCid', '=', dataCid)
+            .executeTakeFirst();
+        if (!ref) {
+            return undefined;
+        }
+        const response = await this.#s3.send(new GetObjectCommand({
+            Bucket: this.#bucket,
+            Key: dataCid,
+        }));
+        if (!response.Body) {
+            return undefined;
+        }
+        const dataStream = response.Body.transformToWebStream();
+        return {
+            dataSize: Number(ref.dataSize),
+            dataStream,
+        };
+    }
+    async put(tenant, recordId, dataCid, dataStream) {
+        const db = this.#getDb('put');
+        // Check if this exact ref already exists (idempotent put).
+        const existingRef = await db
+            .selectFrom('dataRefs')
+            .select('dataSize')
+            .where('tenant', '=', tenant)
+            .where('recordId', '=', recordId)
+            .where('dataCid', '=', dataCid)
+            .executeTakeFirst();
+        if (existingRef) {
+            await DataStream.toBytes(dataStream);
+            return { dataSize: Number(existingRef.dataSize) };
+        }
+        // Check if another ref for this dataCid already exists (dedup path).
+        const otherRef = await db
+            .selectFrom('dataRefs')
+            .select('dataSize')
+            .where('dataCid', '=', dataCid)
+            .executeTakeFirst();
+        let dataSize;
+        if (otherRef) {
+            // S3 object already exists — skip upload.
+            await DataStream.toBytes(dataStream);
+            dataSize = Number(otherRef.dataSize);
+        }
+        else {
+            // New data — upload to S3 with a counting passthrough.
+            dataSize = await this.#uploadToS3(dataCid, dataStream);
+        }
+        // Insert the reference.
+        await db
+            .insertInto('dataRefs')
+            .values({ tenant, recordId, dataCid, dataSize })
+            .execute();
+        return { dataSize };
+    }
+    async delete(tenant, recordId, dataCid) {
+        const db = this.#getDb('delete');
+        // Remove the reference.
+        await db
+            .deleteFrom('dataRefs')
+            .where('tenant', '=', tenant)
+            .where('recordId', '=', recordId)
+            .where('dataCid', '=', dataCid)
+            .execute();
+        // Garbage-collect the S3 object if no more refs point to this dataCid.
+        const remaining = await db
+            .selectFrom('dataRefs')
+            .select('dataCid')
+            .where('dataCid', '=', dataCid)
+            .executeTakeFirst();
+        if (!remaining) {
+            await this.#s3.send(new DeleteObjectCommand({
+                Bucket: this.#bucket,
+                Key: dataCid,
+            }));
+        }
+    }
+    async clear() {
+        const db = this.#getDb('clear');
+        // Clear the refs table.
+        await db.deleteFrom('dataRefs').execute();
+        // Delete all S3 objects in the bucket.
+        let continuationToken;
+        do {
+            const list = await this.#s3.send(new ListObjectsV2Command({
+                Bucket: this.#bucket,
+                ContinuationToken: continuationToken,
+            }));
+            const objects = (list.Contents ?? [])
+                .filter((obj) => obj.Key !== undefined)
+                .map((obj) => ({ Key: obj.Key }));
+            if (objects.length > 0) {
+                await this.#s3.send(new DeleteObjectsCommand({
+                    Bucket: this.#bucket,
+                    Delete: { Objects: objects },
+                }));
+            }
+            continuationToken = list.NextContinuationToken;
+        } while (continuationToken);
+    }
+    // ─── Private helpers ────────────────────────────────────────────────
+    #getDb(method) {
+        if (!this.#db) {
+            throw new Error(`Connection to database not open. Call \`open\` before using \`${method}\`.`);
+        }
+        return this.#db;
+    }
+    /**
+     * Uploads data to S3, counting bytes as they stream through.
+     * Uses multipart upload for large files via `@aws-sdk/lib-storage`.
+     * @returns The total number of bytes uploaded.
+     */
+    async #uploadToS3(dataCid, dataStream) {
+        let dataSize = 0;
+        // Create a Node Readable from the web ReadableStream, counting bytes.
+        const reader = dataStream.getReader();
+        const nodeStream = new Readable({
+            async read() {
+                const { done, value } = await reader.read();
+                if (done) {
+                    this.push(null);
+                }
+                else {
+                    dataSize += value.byteLength;
+                    this.push(Buffer.from(value));
+                }
+            },
+        });
+        // For small files, a simple PutObject suffices. For large files,
+        // Upload handles multipart automatically with bounded memory.
+        if (this.#partSize > 0) {
+            const upload = new Upload({
+                client: this.#s3,
+                params: {
+                    Bucket: this.#bucket,
+                    Key: dataCid,
+                    Body: nodeStream,
+                },
+                queueSize: this.#queueSize,
+                partSize: this.#partSize,
+            });
+            await upload.done();
+        }
+        else {
+            // Fallback: buffer entire stream (only for tiny test payloads).
+            const chunks = [];
+            for (;;) {
+                const { done, value } = await reader.read();
+                if (done) {
+                    break;
+                }
+                dataSize += value.byteLength;
+                chunks.push(value);
+            }
+            const body = Buffer.concat(chunks);
+            await this.#s3.send(new PutObjectCommand({
+                Bucket: this.#bucket,
+                Key: dataCid,
+                Body: body,
+            }));
+        }
+        return dataSize;
+    }
+    /**
+     * Creates the `dataRefs` table if it doesn't already exist.
+     * Shares the same schema as DataStoreSql's `dataRefs` table.
+     */
+    async #ensureRefsTable() {
+        const db = this.#db;
+        if (!(await this.#dialect.hasTable(db, 'dataRefs'))) {
+            await db.schema
+                .createTable('dataRefs')
+                .ifNotExists()
+                .addColumn('tenant', 'varchar(255)', (col) => col.notNull())
+                .addColumn('recordId', 'varchar(60)', (col) => col.notNull())
+                .addColumn('dataCid', 'varchar(60)', (col) => col.notNull())
+                .addColumn('dataSize', 'bigint', (col) => col.notNull())
+                .execute();
+            await db.schema.createIndex('index_dataRefs_tenant_recordId_dataCid')
+                .on('dataRefs').columns(['tenant', 'recordId', 'dataCid']).unique().execute();
+            await db.schema.createIndex('index_dataRefs_dataCid')
+                .on('dataRefs').column('dataCid').execute();
+            await db.schema.createIndex('index_dataRefs_tenant')
+                .on('dataRefs').column('tenant').execute();
+        }
+    }
+}
+//# sourceMappingURL=data-store-s3.js.map

package/dist/esm/src/data-store-s3.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"data-store-s3.js","sourceRoot":"","sources":["../../../src/data-store-s3.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAC9C,OAAO,EACL,mBAAmB,EACnB,oBAAoB,EACpB,gBAAgB,EAChB,oBAAoB,EACpB,gBAAgB,EAChB,QAAQ,GACT,MAAM,oBAAoB,CAAC;AAE5B;;;;;;;;;;;;GAYG;AACH,MAAM,OAAO,WAAW;IACtB,QAAQ,CAAU;IAClB,GAAG,GAAmC,IAAI,CAAC;IAC3C,GAAG,CAAW;IACd,OAAO,CAAS;IAChB,SAAS,CAAS;IAClB,UAAU,CAAS;IAEnB,YAAY,MAAyB;QACnC,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC;QAC/B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC;QAC7B,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,QAAQ,IAAI,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO;QAC5D,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,SAAS,IAAI,CAAC,CAAC;QAExC,IAAI,CAAC,GAAG,GAAG,MAAM,CAAC,QAAQ,IAAI,IAAI,QAAQ,CAAC;YACzC,MAAM,EAAW,MAAM,CAAC,MAAM,IAAI,WAAW;YAC7C,QAAQ,EAAS,MAAM,CAAC,QAAQ;YAChC,cAAc,EAAG,MAAM,CAAC,cAAc,IAAI,KAAK;YAC/C,WAAW,EAAM,MAAM,CAAC,WAAW;SACpC,CAAC,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,IAAI;QACf,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;YACb,OAAO;QACT,CAAC;QAED,IAAI,CAAC,GAAG,GAAG,IAAI,MAAM,CAAkB,EAAE,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QACnE,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAChC,CAAC;IAEM,KAAK,CAAC,KAAK;QAChB,MAAM,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,CAAC;QAC1B,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC;IAClB,CAAC;IAEM,KAAK,CAAC,GAAG,CACd,MAAc,EACd,QAAgB,EAChB,OAAe;QAEf,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAE9B,MAAM,GAAG,GAAG,MAAM,EAAE;aACjB,UAAU,CAAC,UAAU,CAAC;aACtB,MAAM,CAAC,UAAU,CAAC;aAClB,KAAK,CAAC,QAAQ,EAAE,GAAG,EAAE,MAAM,CAAC;aAC5B,KAAK,CAAC,UAAU,EAAE,GAAG,EAAE,QAAQ,CAAC;aAChC,KAAK,CAAC,SAAS,EAAE,GAAG,EAAE,OAAO,CAAC;aAC9B,gBAAgB,EAAE,CAAC;QAEtB,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,gBAAgB,CAAC;YACxD,MAAM,EAAG,IAAI,CAAC,OAAO;YACrB,GAAG,EAAM,OAAO;SACjB,CAAC,CAAC,CAAC;QAEJ,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnB,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,MAAM,UAAU,GAAG,QAAQ,CAAC,IAAI,CAAC,oBAAoB,EAAgC,CAAC;QAEtF,OAAO;YACL,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC9B,UAAU;SACX,CAAC;IACJ,CAAC;IAEM,KAAK,CAAC,GAAG,CACd,MAAc,EACd,QAAgB,EAChB,OAAe,EACf,UAAsC;QAEtC,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAE9B,2DAA2D;QAC3D,MAAM,WAAW,GAAG,MAAM,EAAE;aACzB,UAAU,CAAC,UAAU,CAAC;aACtB,MAAM,CAAC,UAAU,CAAC;aAClB,KAAK,CAAC,QAAQ,EAAE,GAAG,EAAE,MAAM,CAAC;aAC5B,KAAK,CAAC,UAAU,EAAE,GAAG,EAAE,QAAQ,CAAC;aAChC,KAAK,CAAC,SAAS,EAAE,GAAG,EAAE,OAAO,CAAC;aAC9B,gBAAgB,EAAE,CAAC;QAEtB,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,UAAU,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;YACrC,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;QACpD,CAAC;QAED,qEAAqE;QACrE,MAAM,QAAQ,GAAG,MAAM,EAAE;aACtB,UAAU,CAAC,UAAU,CAAC;aACtB,MAAM,CAAC,UAAU,CAAC;aAClB,KAAK,CAAC,SAAS,EAAE,GAAG,EAAE,OAAO,CAAC;aAC9B,gBAAgB,EAAE,CAAC;QAEtB,IAAI,QAAgB,CAAC;QAErB,IAAI,QAAQ,EAAE,CAAC;YACb,0CAA0C;YAC1C,MAAM,UAAU,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;YACrC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACvC,CAAC;aAAM,CAAC;YACN,uDAAuD;YACvD,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;QACzD,CAAC;QAED,wBAAwB;QACxB,MAAM,EAAE;aACL,UAAU,CAAC,UAAU,CAAC;aACtB,MAAM,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;aAC/C,OAAO,EAAE,CAAC;QAEb,OAAO,EAAE,QAAQ,EAAE,CAAC;IACtB,CAAC;IAEM,KAAK,CAAC,MAAM,CACjB,MAAc,EACd,QAAgB,EAChB,OAAe;QAEf,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAEjC,wBAAwB;QACxB,MAAM,EAAE;aACL,UAAU,CAAC,UAAU,CAAC;aACtB,KAAK,CAAC,QAAQ,EAAE,GAAG,EAAE,MAAM,CAAC;aAC5B,KAAK,CAAC,UAAU,EAAE,GAAG,EAAE,QAAQ,CAAC;aAChC,KAAK,CAAC,SAAS,EAAE,GAAG,EAAE,OAAO,CAAC;aAC9B,OAAO,EAAE,CAAC;QAEb,uEAAuE;QACvE,MAAM,SAAS,GAAG,MAAM,EAAE;aACvB,UAAU,CAAC,UAAU,CAAC;aACtB,MAAM,CAAC,SAAS,CAAC;aACjB,KAAK,CAAC,SAAS,EAAE,GAAG,EAAE,OAAO,CAAC;aAC9B,gBAAgB,EAAE,CAAC;QAEtB,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,mBAAmB,CAAC;gBAC1C,MAAM,EAAG,IAAI,CAAC,OAAO;gBACrB,GAAG,EAAM,OAAO;aACjB,CAAC,CAAC,CAAC;QACN,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,KAAK;QAChB,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAEhC,wBAAwB;QACxB,MAAM,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,OAAO,EAAE,CAAC;QAE1C,uCAAuC;QACvC,IAAI,iBAAqC,CAAC;QAC1C,GAAG,CAAC;YACF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,oBAAoB,CAAC;gBACxD,MAAM,EAAc,IAAI,CAAC,OAAO;gBAChC,iBAAiB,EAAG,iBAAiB;aACtC,CAAC,CAAC,CAAC;YAEJ,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;iBAClC,MAAM,CAAC,CAAC,GAAG,EAA0B,EAAE,CAAC,GAAG,CAAC,GAAG,KAAK,SAAS,CAAC;iBAC9D,GAAG,CAAC,CAAC,GAAG,EAAmB,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;YAErD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,oBAAoB,CAAC;oBAC3C,MAAM,EAAG,IAAI,CAAC,OAAO;oBACrB,MAAM,EAAG,EAAE,OAAO,EAAE,OAAO,EAAE;iBAC9B,CAAC,CAAC,CAAC;YACN,CAAC;YAED,iBAAiB,GAAG,IAAI,CAAC,qBAAqB,CAAC;QACjD,CAAC,QAAQ,iBAAiB,EAAE;IAC9B,CAAC;IAED,uEAAuE;IAEvE,MAAM,CAAC,MAAc;QACnB,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CACb,iEAAiE,MAAM,KAAK,CAC7E,CAAC;QACJ,CAAC;QACD,OAAO,IAAI,CAAC,GAAG,CAAC;IAClB,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,WAAW,CAAC,OAAe,EAAE,UAAsC;QACvE,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,sEAAsE;QACtE,MAAM,MAAM,GAAG,UAAU,CAAC,SAAS,EAAE,CAAC;QACtC,MAAM,UAAU,GAAG,IAAI,QAAQ,CAAC;YAC9B,KAAK,CAAC,IAAI;gBACR,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC5C,IAAI,IAAI,EAAE,CAAC;oBACT,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAClB,CAAC;qBAAM,CAAC;oBACN,QAAQ,IAAI,KAAK,CAAC,UAAU,CAAC;oBAC7B,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;gBAChC,CAAC;YACH,CAAC;SACF,CAAC,CAAC;QAEH,iEAAiE;QACjE,8DAA8D;QAC9D,IAAI,IAAI,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC;gBACxB,MAAM,EAAG,IAAI,CAAC,GAAG;gBACjB,MAAM,EAAG;oBACP,MAAM,EAAG,IAAI,CAAC,OAAO;oBACrB,GAAG,EAAM,OAAO;oBAChB,IAAI,EAAK,UAAU;iBACpB;gBACD,SAAS,EAAG,IAAI,CAAC,UAAU;gBAC3B,QAAQ,EAAI,IAAI,CAAC,SAAS;aAC3B,CAAC,CAAC;YAEH,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACtB,CAAC;aAAM,CAAC;YACN,gEAAgE;YAChE,MAAM,MAAM,GAAiB,EAAE,CAAC;YAChC,SAAS,CAAC;gBACR,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC5C,IAAI,IAAI,EAAE,CAAC;oBAAC,MAAM;gBAAC,CAAC;gBACpB,QAAQ,IAAI,KAAK,CAAC,UAAU,CAAC;gBAC7B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC;YACD,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACnC,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,gBAAgB,CAAC;gBACvC,MAAM,EAAG,IAAI,CAAC,OAAO;gBACrB,GAAG,EAAM,OAAO;gBAChB,IAAI,EAAK,IAAI;aACd,CAAC,CAAC,CAAC;QACN,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,gBAAgB;QACpB,MAAM,EAAE,GAAG,IAAI,CAAC,GAAI,CAAC;QAErB,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC,EAAE,CAAC;YACpD,MAAM,EAAE,CAAC,MAAM;iBACZ,WAAW,CAAC,UAAU,CAAC;iBACvB,WAAW,EAAE;iBACb,SAAS,CAAC,QAAQ,EAAE,cAAc,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;iBAC3D,SAAS,CAAC,UAAU,EAAE,aAAa,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;iBAC5D,SAAS,CAAC,SAAS,EAAE,aAAa,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;iBAC3D,SAAS,CAAC,UAAU,EAAE,QAAQ,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;iBACvD,OAAO,EAAE,CAAC;YAEb,MAAM,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,wCAAwC,CAAC;iBAClE,EAAE,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,EAAE,CAAC;YAEhF,MAAM,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,wBAAwB,CAAC;iBAClD,EAAE,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC;YAE9C,MAAM,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,uBAAuB,CAAC;iBACjD,EAAE,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC;QAC/C,CAAC;IACH,CAAC;CACF"}

package/dist/esm/src/data-store-sql.js

@@ -1,5 +1,21 @@
+import { BlockstoreSql } from './blockstore-sql.js';
+import { CID } from 'multiformats';
 import { DataStream } from '@enbox/dwn-sdk-js';
+import { exporter } from 'ipfs-unixfs-exporter';
+import { importer } from 'ipfs-unixfs-importer';
 import { Kysely } from 'kysely';
+/**
+ * SQL-backed implementation of {@link DataStore} with content-addressed
+ * deduplication.
+ *
+ * Data is stored as DAG-PB blocks (via `ipfs-unixfs-importer`) in the
+ * `dataBlocks` table, keyed by `(rootDataCid, blockCid)`. A separate
+ * `dataRefs` table maps `(tenant, recordId, dataCid)` to content. When
+ * multiple records share the same `dataCid`, blocks are stored only once.
+ *
+ * On `delete()`, the ref is removed and blocks are garbage-collected only
+ * when the last ref to a `dataCid` is gone.
+ */
 export class DataStoreSql {
     #dialect;
     #db = null;
@@ -11,93 +27,193 @@ export class DataStoreSql {
             return;
         }
         this.#db = new Kysely({ dialect: this.#dialect });
-        // if table already exists, there is no more things todo
-        const tableName = 'dataStore';
-        const tableExists = await this.#dialect.hasTable(this.#db, tableName);
-        if (tableExists) {
-            return;
-        }
-        // else create the table and corresponding indexes
-        let table = this.#db.schema
-            .createTable(tableName)
-            .ifNotExists() // kept to show supported by all dialects in contrast to ifNotExists() below, though not needed due to hasTable() check above
-            .addColumn('tenant', 'varchar(255)', (col) => col.notNull())
-            .addColumn('recordId', 'varchar(60)', (col) => col.notNull())
-            .addColumn('dataCid', 'varchar(60)', (col) => col.notNull());
-        // Add columns that have dialect-specific constraints
-        table = this.#dialect.addAutoIncrementingColumn(table, 'id', (col) => col.primaryKey());
-        table = this.#dialect.addBlobColumn(table, 'data', (col) => col.notNull());
-        await table.execute();
-        // Add index for efficient lookups.
-        await this.#db.schema
-            .createIndex('tenant_recordId_dataCid')
-            // .ifNotExists() // intentionally kept commented out code to show that it is not supported by all dialects (ie. MySQL)
-            .on(tableName)
-            .columns(['tenant', 'recordId', 'dataCid'])
-            .unique()
-            .execute();
+        // Create tables if they don't exist. In production the MigrationRunner
+        // creates these before open() is called; this fallback handles standalone
+        // usage (tests, plugins) that bypass the migration runner.
+        await this.#ensureTables();
     }
     async close() {
         await this.#db?.destroy();
         this.#db = null;
     }
     async get(tenant, recordId, dataCid) {
-        if (!this.#db) {
-            throw new Error('Connection to database not open. Call `open` before using `get`.');
-        }
-        const result = await this.#db
-            .selectFrom('dataStore')
-            .selectAll()
+        const db = this.#getDb('get');
+        // Look up the reference to confirm this tenant+record has this data.
+        const ref = await db
+            .selectFrom('dataRefs')
+            .select('dataSize')
             .where('tenant', '=', tenant)
             .where('recordId', '=', recordId)
             .where('dataCid', '=', dataCid)
             .executeTakeFirst();
-        if (!result) {
+        if (!ref) {
             return undefined;
         }
-        const dataBytes = new Uint8Array(result.data);
-        return {
-            dataSize: result.data.length,
-            dataStream: new ReadableStream({
-                start(controller) {
-                    controller.enqueue(dataBytes);
+        const blockstore = new BlockstoreSql(db, dataCid);
+        // Use ipfs-unixfs-exporter to stream data from DAG-PB blocks.
+        const dataDagRoot = await exporter(dataCid, blockstore);
+        const contentIterator = dataDagRoot.content();
+        const dataStream = new ReadableStream({
+            async pull(controller) {
+                const result = await contentIterator.next();
+                if (result.done) {
                     controller.close();
                 }
-            }),
+                else {
+                    controller.enqueue(result.value);
+                }
+            },
+        });
+        return {
+            dataSize: Number(ref.dataSize),
+            dataStream,
         };
     }
     async put(tenant, recordId, dataCid, dataStream) {
-        if (!this.#db) {
-            throw new Error('Connection to database not open. Call `open` before using `put`.');
+        const db = this.#getDb('put');
+        // Check if this exact ref already exists (idempotent put).
+        const existingRef = await db
+            .selectFrom('dataRefs')
+            .select('dataSize')
+            .where('tenant', '=', tenant)
+            .where('recordId', '=', recordId)
+            .where('dataCid', '=', dataCid)
+            .executeTakeFirst();
+        if (existingRef) {
+            // Drain the stream — caller expects it to be consumed.
+            await DataStream.toBytes(dataStream);
+            return { dataSize: Number(existingRef.dataSize) };
         }
-        const bytes = await DataStream.toBytes(dataStream);
-        const data = Buffer.from(bytes);
-        await this.#db
-            .insertInto('dataStore')
-            .values({ tenant, recordId, dataCid, data })
-            .executeTakeFirstOrThrow();
-        return {
-            dataSize: bytes.length
-        };
+        // Check if blocks for this dataCid already exist (dedup path).
+        const blockstore = new BlockstoreSql(db, dataCid);
+        const rootCid = CID.parse(dataCid);
+        const blocksExist = await blockstore.has(rootCid);
+        let dataSize;
+        if (blocksExist) {
+            // Blocks already stored by a previous ref with the same dataCid.
+            // Get the data size from that existing ref.
+            const otherRef = await db
+                .selectFrom('dataRefs')
+                .select('dataSize')
+                .where('dataCid', '=', dataCid)
+                .executeTakeFirst();
+            if (otherRef) {
+                dataSize = Number(otherRef.dataSize);
+                // Drain the stream — caller expects it to be consumed.
+                await DataStream.toBytes(dataStream);
+            }
+            else {
+                // Edge case: blocks exist but no ref (interrupted previous put).
+                // Count bytes without full buffering.
+                dataSize = await DataStoreSql.#countStreamBytes(dataStream);
+            }
+        }
+        else {
+            // New data — clean up any partial blocks from interrupted imports,
+            // then chunk the data into DAG-PB blocks via the importer.
+            await blockstore.clear();
+            const asyncDataBlocks = importer([{ content: DataStream.asAsyncIterable(dataStream) }], blockstore, { cidVersion: 1 });
+            // The last block yielded contains the root CID and file size info.
+            let dataDagRoot;
+            for await (dataDagRoot of asyncDataBlocks) {
+                ;
+            }
+            dataSize = Number(dataDagRoot.unixfs?.fileSize() ?? dataDagRoot.size);
+        }
+        // Insert the reference.
+        await db
+            .insertInto('dataRefs')
+            .values({ tenant, recordId, dataCid, dataSize })
+            .execute();
+        return { dataSize };
     }
     async delete(tenant, recordId, dataCid) {
-        if (!this.#db) {
-            throw new Error('Connection to database not open. Call `open` before using `delete`.');
-        }
-        await this.#db
-            .deleteFrom('dataStore')
+        const db = this.#getDb('delete');
+        // Remove the reference.
+        await db
+            .deleteFrom('dataRefs')
             .where('tenant', '=', tenant)
             .where('recordId', '=', recordId)
             .where('dataCid', '=', dataCid)
             .execute();
+        // Garbage-collect blocks if no more refs point to this dataCid.
+        const remaining = await db
+            .selectFrom('dataRefs')
+            .select('dataCid')
+            .where('dataCid', '=', dataCid)
+            .executeTakeFirst();
+        if (!remaining) {
+            await db
+                .deleteFrom('dataBlocks')
+                .where('rootDataCid', '=', dataCid)
+                .execute();
+        }
     }
     async clear() {
+        const db = this.#getDb('clear');
+        await db.deleteFrom('dataRefs').execute();
+        await db.deleteFrom('dataBlocks').execute();
+    }
+    // ─── Private helpers ────────────────────────────────────────────────
+    /**
+     * Returns the open database instance, or throws if not yet opened.
+     */
+    #getDb(method) {
         if (!this.#db) {
-            throw new Error('Connection to database not open. Call `open` before using `clear`.');
+            throw new Error(`Connection to database not open. Call \`open\` before using \`${method}\`.`);
         }
-        await this.#db
-            .deleteFrom('dataStore')
-            .execute();
+        return this.#db;
+    }
+    /**
+     * Creates the `dataRefs` and `dataBlocks` tables if they don't already exist.
+     * This is a fallback for standalone usage without the MigrationRunner.
+     */
+    async #ensureTables() {
+        const db = this.#db;
+        // ─── dataRefs ─────────────────────────────────────────────────────
+        if (!(await this.#dialect.hasTable(db, 'dataRefs'))) {
+            await db.schema
+                .createTable('dataRefs')
+                .ifNotExists()
+                .addColumn('tenant', 'varchar(255)', (col) => col.notNull())
+                .addColumn('recordId', 'varchar(60)', (col) => col.notNull())
+                .addColumn('dataCid', 'varchar(60)', (col) => col.notNull())
+                .addColumn('dataSize', 'bigint', (col) => col.notNull())
+                .execute();
+            await db.schema.createIndex('index_dataRefs_tenant_recordId_dataCid')
+                .on('dataRefs').columns(['tenant', 'recordId', 'dataCid']).unique().execute();
+            await db.schema.createIndex('index_dataRefs_dataCid')
+                .on('dataRefs').column('dataCid').execute();
+            await db.schema.createIndex('index_dataRefs_tenant')
+                .on('dataRefs').column('tenant').execute();
+        }
+        // ─── dataBlocks ───────────────────────────────────────────────────
+        if (!(await this.#dialect.hasTable(db, 'dataBlocks'))) {
+            let table = db.schema
+                .createTable('dataBlocks')
+                .ifNotExists()
+                .addColumn('rootDataCid', 'varchar(60)', (col) => col.notNull())
+                .addColumn('blockCid', 'varchar(60)', (col) => col.notNull());
+            table = this.#dialect.addBlobColumn(table, 'data', (col) => col.notNull());
+            await table.execute();
+            await db.schema.createIndex('index_dataBlocks_rootDataCid_blockCid')
+                .on('dataBlocks').columns(['rootDataCid', 'blockCid']).unique().execute();
+        }
+    }
+    /**
+     * Counts the number of bytes in a stream without buffering the full content.
+     */
+    static async #countStreamBytes(stream) {
+        const reader = stream.getReader();
+        let size = 0;
+        for (;;) {
+            const { done, value } = await reader.read();
+            if (done) {
+                break;
+            }
+            size += value.byteLength;
+        }
+        return size;
     }
 }
 //# sourceMappingURL=data-store-sql.js.map