@enbox/dwn-sql-store 0.0.7 → 0.0.8

package/src/data-store-s3.ts

@@ -0,0 +1,338 @@
+import type { Dialect } from './dialect/dialect.js';
+import type { DwnDatabaseType } from './types.js';
+import type { DataStore, DataStoreGetResult, DataStorePutResult } from '@enbox/dwn-sdk-js';
+
+import { DataStream } from '@enbox/dwn-sdk-js';
+import { Kysely } from 'kysely';
+import { Readable } from 'stream';
+import { Upload } from '@aws-sdk/lib-storage';
+import {
+  DeleteObjectCommand,
+  DeleteObjectsCommand,
+  GetObjectCommand,
+  ListObjectsV2Command,
+  PutObjectCommand,
+  S3Client,
+} from '@aws-sdk/client-s3';
+
+/**
+ * S3-backed implementation of {@link DataStore} with SQL-based reference
+ * tracking for content-addressed deduplication.
+ *
+ * Data is stored as whole S3 objects keyed by `dataCid`. The same `dataCid`
+ * maps to a single S3 object regardless of how many (tenant, recordId) pairs
+ * reference it. A `dataRefs` SQL table tracks references; blocks are
+ * garbage-collected from S3 when the last ref is deleted.
+ *
+ * For files over `partSize` (default 5MB), the AWS SDK Upload helper
+ * automatically uses multipart upload with bounded memory
+ * (`queueSize * partSize`).
+ */
+export class DataStoreS3 implements DataStore {
+  #dialect: Dialect;
+  #db: Kysely<DwnDatabaseType> | null = null;
+  #s3: S3Client;
+  #bucket: string;
+  #partSize: number;
+  #queueSize: number;
+
+  constructor(config: DataStoreS3Config) {
+    this.#dialect = config.dialect;
+    this.#bucket = config.bucket;
+    this.#partSize = config.partSize ?? 5 * 1024 * 1024; // 5 MB
+    this.#queueSize = config.queueSize ?? 4;
+
+    this.#s3 = config.s3Client ?? new S3Client({
+      region         : config.region ?? 'us-east-1',
+      endpoint       : config.endpoint,
+      forcePathStyle : config.forcePathStyle ?? false,
+      credentials    : config.credentials,
+    });
+  }
+
+  public async open(): Promise<void> {
+    if (this.#db) {
+      return;
+    }
+
+    this.#db = new Kysely<DwnDatabaseType>({ dialect: this.#dialect });
+    await this.#ensureRefsTable();
+  }
+
+  public async close(): Promise<void> {
+    await this.#db?.destroy();
+    this.#db = null;
+  }
+
+  public async get(
+    tenant: string,
+    recordId: string,
+    dataCid: string,
+  ): Promise<DataStoreGetResult | undefined> {
+    const db = this.#getDb('get');
+
+    const ref = await db
+      .selectFrom('dataRefs')
+      .select('dataSize')
+      .where('tenant', '=', tenant)
+      .where('recordId', '=', recordId)
+      .where('dataCid', '=', dataCid)
+      .executeTakeFirst();
+
+    if (!ref) {
+      return undefined;
+    }
+
+    const response = await this.#s3.send(new GetObjectCommand({
+      Bucket : this.#bucket,
+      Key    : dataCid,
+    }));
+
+    if (!response.Body) {
+      return undefined;
+    }
+
+    const dataStream = response.Body.transformToWebStream() as ReadableStream<Uint8Array>;
+
+    return {
+      dataSize: Number(ref.dataSize),
+      dataStream,
+    };
+  }
+
+  public async put(
+    tenant: string,
+    recordId: string,
+    dataCid: string,
+    dataStream: ReadableStream<Uint8Array>,
+  ): Promise<DataStorePutResult> {
+    const db = this.#getDb('put');
+
+    // Check if this exact ref already exists (idempotent put).
+    const existingRef = await db
+      .selectFrom('dataRefs')
+      .select('dataSize')
+      .where('tenant', '=', tenant)
+      .where('recordId', '=', recordId)
+      .where('dataCid', '=', dataCid)
+      .executeTakeFirst();
+
+    if (existingRef) {
+      await DataStream.toBytes(dataStream);
+      return { dataSize: Number(existingRef.dataSize) };
+    }
+
+    // Check if another ref for this dataCid already exists (dedup path).
+    const otherRef = await db
+      .selectFrom('dataRefs')
+      .select('dataSize')
+      .where('dataCid', '=', dataCid)
+      .executeTakeFirst();
+
+    let dataSize: number;
+
+    if (otherRef) {
+      // S3 object already exists — skip upload.
+      await DataStream.toBytes(dataStream);
+      dataSize = Number(otherRef.dataSize);
+    } else {
+      // New data — upload to S3 with a counting passthrough.
+      dataSize = await this.#uploadToS3(dataCid, dataStream);
+    }
+
+    // Insert the reference.
+    await db
+      .insertInto('dataRefs')
+      .values({ tenant, recordId, dataCid, dataSize })
+      .execute();
+
+    return { dataSize };
+  }
+
+  public async delete(
+    tenant: string,
+    recordId: string,
+    dataCid: string,
+  ): Promise<void> {
+    const db = this.#getDb('delete');
+
+    // Remove the reference.
+    await db
+      .deleteFrom('dataRefs')
+      .where('tenant', '=', tenant)
+      .where('recordId', '=', recordId)
+      .where('dataCid', '=', dataCid)
+      .execute();
+
+    // Garbage-collect the S3 object if no more refs point to this dataCid.
+    const remaining = await db
+      .selectFrom('dataRefs')
+      .select('dataCid')
+      .where('dataCid', '=', dataCid)
+      .executeTakeFirst();
+
+    if (!remaining) {
+      await this.#s3.send(new DeleteObjectCommand({
+        Bucket : this.#bucket,
+        Key    : dataCid,
+      }));
+    }
+  }
+
+  public async clear(): Promise<void> {
+    const db = this.#getDb('clear');
+
+    // Clear the refs table.
+    await db.deleteFrom('dataRefs').execute();
+
+    // Delete all S3 objects in the bucket.
+    let continuationToken: string | undefined;
+    do {
+      const list = await this.#s3.send(new ListObjectsV2Command({
+        Bucket            : this.#bucket,
+        ContinuationToken : continuationToken,
+      }));
+
+      const objects = (list.Contents ?? [])
+        .filter((obj): obj is { Key: string } => obj.Key !== undefined)
+        .map((obj): { Key: string } => ({ Key: obj.Key }));
+
+      if (objects.length > 0) {
+        await this.#s3.send(new DeleteObjectsCommand({
+          Bucket : this.#bucket,
+          Delete : { Objects: objects },
+        }));
+      }
+
+      continuationToken = list.NextContinuationToken;
+    } while (continuationToken);
+  }
+
+  // ─── Private helpers ────────────────────────────────────────────────
+
+  #getDb(method: string): Kysely<DwnDatabaseType> {
+    if (!this.#db) {
+      throw new Error(
+        `Connection to database not open. Call \`open\` before using \`${method}\`.`
+      );
+    }
+    return this.#db;
+  }
+
+  /**
+   * Uploads data to S3, counting bytes as they stream through.
+   * Uses multipart upload for large files via `@aws-sdk/lib-storage`.
+   * @returns The total number of bytes uploaded.
+   */
+  async #uploadToS3(dataCid: string, dataStream: ReadableStream<Uint8Array>): Promise<number> {
+    let dataSize = 0;
+
+    // Create a Node Readable from the web ReadableStream, counting bytes.
+    const reader = dataStream.getReader();
+    const nodeStream = new Readable({
+      async read(): Promise<void> {
+        const { done, value } = await reader.read();
+        if (done) {
+          this.push(null);
+        } else {
+          dataSize += value.byteLength;
+          this.push(Buffer.from(value));
+        }
+      },
+    });
+
+    // For small files, a simple PutObject suffices. For large files,
+    // Upload handles multipart automatically with bounded memory.
+    if (this.#partSize > 0) {
+      const upload = new Upload({
+        client : this.#s3,
+        params : {
+          Bucket : this.#bucket,
+          Key    : dataCid,
+          Body   : nodeStream,
+        },
+        queueSize : this.#queueSize,
+        partSize  : this.#partSize,
+      });
+
+      await upload.done();
+    } else {
+      // Fallback: buffer entire stream (only for tiny test payloads).
+      const chunks: Uint8Array[] = [];
+      for (;;) {
+        const { done, value } = await reader.read();
+        if (done) { break; }
+        dataSize += value.byteLength;
+        chunks.push(value);
+      }
+      const body = Buffer.concat(chunks);
+      await this.#s3.send(new PutObjectCommand({
+        Bucket : this.#bucket,
+        Key    : dataCid,
+        Body   : body,
+      }));
+    }
+
+    return dataSize;
+  }
+
+  /**
+   * Creates the `dataRefs` table if it doesn't already exist.
+   * Shares the same schema as DataStoreSql's `dataRefs` table.
+   */
+  async #ensureRefsTable(): Promise<void> {
+    const db = this.#db!;
+
+    if (!(await this.#dialect.hasTable(db, 'dataRefs'))) {
+      await db.schema
+        .createTable('dataRefs')
+        .ifNotExists()
+        .addColumn('tenant', 'varchar(255)', (col) => col.notNull())
+        .addColumn('recordId', 'varchar(60)', (col) => col.notNull())
+        .addColumn('dataCid', 'varchar(60)', (col) => col.notNull())
+        .addColumn('dataSize', 'bigint', (col) => col.notNull())
+        .execute();
+
+      await db.schema.createIndex('index_dataRefs_tenant_recordId_dataCid')
+        .on('dataRefs').columns(['tenant', 'recordId', 'dataCid']).unique().execute();
+
+      await db.schema.createIndex('index_dataRefs_dataCid')
+        .on('dataRefs').column('dataCid').execute();
+
+      await db.schema.createIndex('index_dataRefs_tenant')
+        .on('dataRefs').column('tenant').execute();
+    }
+  }
+}
+
+/**
+ * Configuration for {@link DataStoreS3}.
+ */
+export type DataStoreS3Config = {
+  /** Kysely dialect for the SQL `dataRefs` table. */
+  dialect: Dialect;
+
+  /** S3 bucket name for content storage. */
+  bucket: string;
+
+  /** Optional pre-configured S3Client instance. If omitted, one is created from region/endpoint. */
+  s3Client?: S3Client;
+
+  /** AWS region. Default: `'us-east-1'`. */
+  region?: string;
+
+  /** Custom S3 endpoint URL (e.g. MinIO `http://localhost:9000`). */
+  endpoint?: string;
+
+  /** Use path-style access (`http://host/bucket/key`). Required for MinIO. Default: `false`. */
+  forcePathStyle?: boolean;
+
+  /** AWS credentials. When omitted, the SDK uses the default credential chain (IAM role, env vars, etc.). */
+  credentials?: { accessKeyId: string; secretAccessKey: string };
+
+  /** Multipart upload part size in bytes. Default: `5 * 1024 * 1024` (5 MB). */
+  partSize?: number;
+
+  /** Number of concurrent multipart upload parts. Default: `4`. */
+  queueSize?: number;
+};

package/src/data-store-sql.ts

@@ -1,10 +1,27 @@
 import type { Dialect } from './dialect/dialect.js';
 import type { DwnDatabaseType } from './types.js';
+import type { ImportResult } from 'ipfs-unixfs-importer';
 import type { DataStore, DataStoreGetResult, DataStorePutResult } from '@enbox/dwn-sdk-js';
 
+import { BlockstoreSql } from './blockstore-sql.js';
+import { CID } from 'multiformats';
 import { DataStream } from '@enbox/dwn-sdk-js';
+import { exporter } from 'ipfs-unixfs-exporter';
+import { importer } from 'ipfs-unixfs-importer';
 import { Kysely } from 'kysely';
 
+/**
+ * SQL-backed implementation of {@link DataStore} with content-addressed
+ * deduplication.
+ *
+ * Data is stored as DAG-PB blocks (via `ipfs-unixfs-importer`) in the
+ * `dataBlocks` table, keyed by `(rootDataCid, blockCid)`. A separate
+ * `dataRefs` table maps `(tenant, recordId, dataCid)` to content. When
+ * multiple records share the same `dataCid`, blocks are stored only once.
+ *
+ * On `delete()`, the ref is removed and blocks are garbage-collected only
+ * when the last ref to a `dataCid` is gone.
+ */
 export class DataStoreSql implements DataStore {
   #dialect: Dialect;
   #db: Kysely<DwnDatabaseType> | null = null;
@@ -13,138 +30,250 @@ export class DataStoreSql implements DataStore {
     this.#dialect = dialect;
   }
 
-  async open(): Promise<void> {
+  public async open(): Promise<void> {
     if (this.#db) {
       return;
     }
 
     this.#db = new Kysely<DwnDatabaseType>({ dialect: this.#dialect });
 
-    // if table already exists, there is no more things todo
-    const tableName = 'dataStore';
-    const tableExists = await this.#dialect.hasTable(this.#db, tableName);
-    if (tableExists) {
-      return;
-    }
-
-    // else create the table and corresponding indexes
-
-    let table = this.#db.schema
-      .createTable(tableName)
-      .ifNotExists()// kept to show supported by all dialects in contrast to ifNotExists() below, though not needed due to hasTable() check above
-      .addColumn('tenant', 'varchar(255)', (col) => col.notNull())
-      .addColumn('recordId', 'varchar(60)', (col) => col.notNull())
-      .addColumn('dataCid', 'varchar(60)', (col) => col.notNull());
-
-    // Add columns that have dialect-specific constraints
-    table = this.#dialect.addAutoIncrementingColumn(table, 'id', (col) => col.primaryKey());
-    table = this.#dialect.addBlobColumn(table, 'data', (col) => col.notNull());
-    await table.execute();
-
-    // Add index for efficient lookups.
-    await this.#db.schema
-      .createIndex('tenant_recordId_dataCid')
-      // .ifNotExists() // intentionally kept commented out code to show that it is not supported by all dialects (ie. MySQL)
-      .on(tableName)
-      .columns(['tenant', 'recordId', 'dataCid'])
-      .unique()
-      .execute();
+    // Create tables if they don't exist. In production the MigrationRunner
+    // creates these before open() is called; this fallback handles standalone
+    // usage (tests, plugins) that bypass the migration runner.
+    await this.#ensureTables();
   }
 
-  async close(): Promise<void> {
+  public async close(): Promise<void> {
     await this.#db?.destroy();
     this.#db = null;
   }
 
-  async get(
+  public async get(
     tenant: string,
     recordId: string,
-    dataCid: string
+    dataCid: string,
   ): Promise<DataStoreGetResult | undefined> {
-    if (!this.#db) {
-      throw new Error(
-        'Connection to database not open. Call `open` before using `get`.'
-      );
-    }
+    const db = this.#getDb('get');
 
-    const result = await this.#db
-      .selectFrom('dataStore')
-      .selectAll()
+    // Look up the reference to confirm this tenant+record has this data.
+    const ref = await db
+      .selectFrom('dataRefs')
+      .select('dataSize')
       .where('tenant', '=', tenant)
       .where('recordId', '=', recordId)
       .where('dataCid', '=', dataCid)
       .executeTakeFirst();
 
-    if (!result) {
+    if (!ref) {
       return undefined;
     }
 
-    const dataBytes = new Uint8Array(result.data);
-    return {
-      dataSize   : result.data.length,
-      dataStream : new ReadableStream<Uint8Array>({
-        start(controller): void {
-          controller.enqueue(dataBytes);
+    const blockstore = new BlockstoreSql(db, dataCid);
+
+    // Use ipfs-unixfs-exporter to stream data from DAG-PB blocks.
+    const dataDagRoot = await exporter(dataCid, blockstore);
+    const contentIterator = dataDagRoot.content();
+
+    const dataStream = new ReadableStream<Uint8Array>({
+      async pull(controller): Promise<void> {
+        const result = await contentIterator.next();
+        if (result.done) {
           controller.close();
+        } else {
+          controller.enqueue(result.value);
         }
-      }),
+      },
+    });
+
+    return {
+      dataSize: Number(ref.dataSize),
+      dataStream,
     };
   }
 
-  async put(
+  public async put(
     tenant: string,
     recordId: string,
     dataCid: string,
-    dataStream: ReadableStream<Uint8Array>
+    dataStream: ReadableStream<Uint8Array>,
   ): Promise<DataStorePutResult> {
-    if (!this.#db) {
-      throw new Error(
-        'Connection to database not open. Call `open` before using `put`.'
-      );
+    const db = this.#getDb('put');
+
+    // Check if this exact ref already exists (idempotent put).
+    const existingRef = await db
+      .selectFrom('dataRefs')
+      .select('dataSize')
+      .where('tenant', '=', tenant)
+      .where('recordId', '=', recordId)
+      .where('dataCid', '=', dataCid)
+      .executeTakeFirst();
+
+    if (existingRef) {
+      // Drain the stream — caller expects it to be consumed.
+      await DataStream.toBytes(dataStream);
+      return { dataSize: Number(existingRef.dataSize) };
     }
 
-    const bytes = await DataStream.toBytes(dataStream);
-    const data = Buffer.from(bytes);
+    // Check if blocks for this dataCid already exist (dedup path).
+    const blockstore = new BlockstoreSql(db, dataCid);
+    const rootCid = CID.parse(dataCid);
+    const blocksExist = await blockstore.has(rootCid);
 
-    await this.#db
-      .insertInto('dataStore')
-      .values({ tenant, recordId, dataCid, data })
-      .executeTakeFirstOrThrow();
+    let dataSize: number;
 
-    return {
-      dataSize: bytes.length
-    };
+    if (blocksExist) {
+      // Blocks already stored by a previous ref with the same dataCid.
+      // Get the data size from that existing ref.
+      const otherRef = await db
+        .selectFrom('dataRefs')
+        .select('dataSize')
+        .where('dataCid', '=', dataCid)
+        .executeTakeFirst();
+
+      if (otherRef) {
+        dataSize = Number(otherRef.dataSize);
+        // Drain the stream — caller expects it to be consumed.
+        await DataStream.toBytes(dataStream);
+      } else {
+        // Edge case: blocks exist but no ref (interrupted previous put).
+        // Count bytes without full buffering.
+        dataSize = await DataStoreSql.#countStreamBytes(dataStream);
+      }
+    } else {
+      // New data — clean up any partial blocks from interrupted imports,
+      // then chunk the data into DAG-PB blocks via the importer.
+      await blockstore.clear();
+
+      const asyncDataBlocks = importer(
+        [{ content: DataStream.asAsyncIterable(dataStream) }],
+        blockstore,
+        { cidVersion: 1 },
+      );
+
+      // The last block yielded contains the root CID and file size info.
+      let dataDagRoot!: ImportResult;
+      for await (dataDagRoot of asyncDataBlocks) { ; }
+
+      dataSize = Number(dataDagRoot.unixfs?.fileSize() ?? dataDagRoot.size);
+    }
+
+    // Insert the reference.
+    await db
+      .insertInto('dataRefs')
+      .values({ tenant, recordId, dataCid, dataSize })
+      .execute();
+
+    return { dataSize };
   }
 
-  async delete(
+  public async delete(
     tenant: string,
     recordId: string,
-    dataCid: string
+    dataCid: string,
   ): Promise<void> {
-    if (!this.#db) {
-      throw new Error(
-        'Connection to database not open. Call `open` before using `delete`.'
-      );
-    }
+    const db = this.#getDb('delete');
 
-    await this.#db
-      .deleteFrom('dataStore')
+    // Remove the reference.
+    await db
+      .deleteFrom('dataRefs')
       .where('tenant', '=', tenant)
       .where('recordId', '=', recordId)
       .where('dataCid', '=', dataCid)
       .execute();
+
+    // Garbage-collect blocks if no more refs point to this dataCid.
+    const remaining = await db
+      .selectFrom('dataRefs')
+      .select('dataCid')
+      .where('dataCid', '=', dataCid)
+      .executeTakeFirst();
+
+    if (!remaining) {
+      await db
+        .deleteFrom('dataBlocks')
+        .where('rootDataCid', '=', dataCid)
+        .execute();
+    }
+  }
+
+  public async clear(): Promise<void> {
+    const db = this.#getDb('clear');
+
+    await db.deleteFrom('dataRefs').execute();
+    await db.deleteFrom('dataBlocks').execute();
   }
 
-  async clear(): Promise<void> {
+  // ─── Private helpers ────────────────────────────────────────────────
+
+  /**
+   * Returns the open database instance, or throws if not yet opened.
+   */
+  #getDb(method: string): Kysely<DwnDatabaseType> {
     if (!this.#db) {
       throw new Error(
-        'Connection to database not open. Call `open` before using `clear`.'
+        `Connection to database not open. Call \`open\` before using \`${method}\`.`
       );
     }
+    return this.#db;
+  }
 
-    await this.#db
-      .deleteFrom('dataStore')
-      .execute();
+  /**
+   * Creates the `dataRefs` and `dataBlocks` tables if they don't already exist.
+   * This is a fallback for standalone usage without the MigrationRunner.
+   */
+  async #ensureTables(): Promise<void> {
+    const db = this.#db!;
+
+    // ─── dataRefs ─────────────────────────────────────────────────────
+    if (!(await this.#dialect.hasTable(db, 'dataRefs'))) {
+      await db.schema
+        .createTable('dataRefs')
+        .ifNotExists()
+        .addColumn('tenant', 'varchar(255)', (col) => col.notNull())
+        .addColumn('recordId', 'varchar(60)', (col) => col.notNull())
+        .addColumn('dataCid', 'varchar(60)', (col) => col.notNull())
+        .addColumn('dataSize', 'bigint', (col) => col.notNull())
+        .execute();
+
+      await db.schema.createIndex('index_dataRefs_tenant_recordId_dataCid')
+        .on('dataRefs').columns(['tenant', 'recordId', 'dataCid']).unique().execute();
+
+      await db.schema.createIndex('index_dataRefs_dataCid')
+        .on('dataRefs').column('dataCid').execute();
+
+      await db.schema.createIndex('index_dataRefs_tenant')
+        .on('dataRefs').column('tenant').execute();
+    }
+
+    // ─── dataBlocks ───────────────────────────────────────────────────
+    if (!(await this.#dialect.hasTable(db, 'dataBlocks'))) {
+      let table = db.schema
+        .createTable('dataBlocks')
+        .ifNotExists()
+        .addColumn('rootDataCid', 'varchar(60)', (col) => col.notNull())
+        .addColumn('blockCid', 'varchar(60)', (col) => col.notNull());
+
+      table = this.#dialect.addBlobColumn(table, 'data', (col) => col.notNull());
+      await table.execute();
+
+      await db.schema.createIndex('index_dataBlocks_rootDataCid_blockCid')
+        .on('dataBlocks').columns(['rootDataCid', 'blockCid']).unique().execute();
+    }
   }
 
-}
+  /**
+   * Counts the number of bytes in a stream without buffering the full content.
+   */
+  static async #countStreamBytes(stream: ReadableStream<Uint8Array>): Promise<number> {
+    const reader = stream.getReader();
+    let size = 0;
+    for (;;) {
+      const { done, value } = await reader.read();
+      if (done) {
+        break;
+      }
+      size += value.byteLength;
+    }
+    return size;
+  }
+}