lex-gql-duckdb 0.2.0 → 0.3.0

package/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## 0.3.0
+
+### Minor Changes
+
+- 45bc9df: Add `insertRecordsBatch()` for bulk inserts
+
+  - New `insertRecordsBatch(records)` method that uses a single INSERT statement with multiple VALUES
+  - Fixes memory leak in relay example caused by unbounded `pendingWrites` queue
+
 ## 0.2.0
 
 ### Minor Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "lex-gql-duckdb",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "DuckDB adapter for lex-gql - optimized for analytics queries",
   "type": "module",
   "main": "src/lex-gql-duckdb.js",

package/src/lex-gql-duckdb.d.ts

@@ -27,6 +27,7 @@ export function setupSchema(conn: DuckDBConnection): Promise<void>;
 /**
  * @typedef {Object} Writer
  * @property {(record: RecordInput) => Promise<void>} insertRecord - Insert or replace a record
+ * @property {(records: RecordInput[]) => Promise<void>} insertRecordsBatch - Insert multiple records in a single statement
  * @property {(uri: string) => Promise<void>} deleteRecord - Delete a record by URI
  * @property {(did: string, handle: string) => Promise<void>} upsertActor - Insert or replace an actor
  */
@@ -109,6 +110,10 @@ export type Writer = {
      * - Insert or replace a record
      */
     insertRecord: (record: RecordInput) => Promise<void>;
+    /**
+     * - Insert multiple records in a single statement
+     */
+    insertRecordsBatch: (records: RecordInput[]) => Promise<void>;
     /**
      * - Delete a record by URI
      */

package/src/lex-gql-duckdb.js

@@ -103,6 +103,7 @@ function parseAtUri(uri) {
 /**
  * @typedef {Object} Writer
  * @property {(record: RecordInput) => Promise<void>} insertRecord - Insert or replace a record
+ * @property {(records: RecordInput[]) => Promise<void>} insertRecordsBatch - Insert multiple records in a single statement
  * @property {(uri: string) => Promise<void>} deleteRecord - Delete a record by URI
  * @property {(did: string, handle: string) => Promise<void>} upsertActor - Insert or replace an actor
  */
@@ -141,6 +142,36 @@ export function createWriter(conn) {
       );
     },
 
+    insertRecordsBatch: async (records) => {
+      if (records.length === 0) return;
+
+      const now = new Date().toISOString();
+      const rows = records.map(({ uri, cid, record, indexedAt }) => {
+        const { did, collection, rkey } = parseAtUri(uri);
+        const recordJson = typeof record === 'string' ? record : JSON.stringify(record);
+        return [uri, did, collection, rkey, cid || null, recordJson, indexedAt || now];
+      });
+
+      // Build a single INSERT with multiple VALUES for ~10x faster performance
+      const placeholders = rows.map(() => '(?, ?, ?, ?, ?, ?, ?)').join(', ');
+      const params = rows.flat();
+
+      await conn.run(
+        `
+        INSERT INTO records (uri, did, collection, rkey, cid, record, indexed_at)
+        VALUES ${placeholders}
+        ON CONFLICT (uri) DO UPDATE SET
+          did = EXCLUDED.did,
+          collection = EXCLUDED.collection,
+          rkey = EXCLUDED.rkey,
+          cid = EXCLUDED.cid,
+          record = EXCLUDED.record,
+          indexed_at = EXCLUDED.indexed_at
+      `,
+        ...params
+      );
+    },
+
     deleteRecord: async (uri) => {
       await conn.run('DELETE FROM records WHERE uri = ?', uri);
     },

package/test/lex-gql-duckdb.test.js

@@ -109,6 +109,43 @@ describe('createWriter', () => {
     expect(rows).toHaveLength(1);
     expect(rows[0].handle).toBe('alice.example.com');
   });
+
+  it('inserts multiple records in a batch', async () => {
+    await writer.insertRecordsBatch([
+      { uri: 'at://did:plc:alice/app.bsky.feed.post/1', cid: 'cid1', record: { text: 'Post 1' } },
+      { uri: 'at://did:plc:alice/app.bsky.feed.post/2', cid: 'cid2', record: { text: 'Post 2' } },
+      { uri: 'at://did:plc:bob/app.bsky.feed.post/1', cid: 'cid3', record: { text: 'Post 3' } },
+    ]);
+
+    const rows = await db.all('SELECT * FROM records ORDER BY uri');
+    expect(rows).toHaveLength(3);
+    expect(JSON.parse(rows[0].record)).toEqual({ text: 'Post 1' });
+    expect(JSON.parse(rows[1].record)).toEqual({ text: 'Post 2' });
+    expect(rows[2].did).toBe('did:plc:bob');
+  });
+
+  it('handles batch upsert (updates on conflict)', async () => {
+    await writer.insertRecord({
+      uri: 'at://did:plc:alice/app.bsky.feed.post/1',
+      record: { text: 'Original' },
+    });
+
+    await writer.insertRecordsBatch([
+      { uri: 'at://did:plc:alice/app.bsky.feed.post/1', record: { text: 'Updated' } },
+      { uri: 'at://did:plc:alice/app.bsky.feed.post/2', record: { text: 'New' } },
+    ]);
+
+    const rows = await db.all('SELECT * FROM records ORDER BY uri');
+    expect(rows).toHaveLength(2);
+    expect(JSON.parse(rows[0].record)).toEqual({ text: 'Updated' });
+    expect(JSON.parse(rows[1].record)).toEqual({ text: 'New' });
+  });
+
+  it('handles empty batch gracefully', async () => {
+    await writer.insertRecordsBatch([]);
+    const rows = await db.all('SELECT * FROM records');
+    expect(rows).toHaveLength(0);
+  });
 });
 
 describe('buildWhere', () => {