lex-gql-sqlite 0.2.0 → 0.2.1

package/.claude/settings.local.json

@@ -0,0 +1,9 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(pnpm install)",
+      "Bash(git add:*)",
+      "Bash(git commit:*)"
+    ]
+  }
+}

package/CHANGELOG.md

@@ -1,5 +1,23 @@
 # Changelog
 
+## 0.2.1
+
+### Patch Changes
+
+- 074d06f: Fix cursor pagination with custom sort fields
+
+  Previously, cursor pagination used only the record ID which caused duplicate records when paginating with custom sort fields. Cursors now encode actual sort field values for correct pagination.
+
+  - Encode sort field values + URI tiebreaker in JSON cursor format
+  - Add progressive WHERE clauses for multi-field sorts
+  - Add field name validation to prevent SQL injection
+  - Export centralized DEFAULT_SORT constant from lex-gql
+
+  **Note:** Existing cursors from previous versions will be invalid and pagination will restart from the beginning.
+
+- Updated dependencies [074d06f]
+  - lex-gql@0.2.1
+
 ## 0.2.0
 
 ### Minor Changes

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Chad Miller
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "lex-gql-sqlite",
-  "version": "0.2.0",
+  "version": "0.2.1",
   "description": "SQLite adapter for lex-gql",
   "type": "module",
   "main": "src/lex-gql-sqlite.js",
@@ -11,12 +11,6 @@
       "default": "./src/lex-gql-sqlite.js"
     }
   },
-  "scripts": {
-    "build": "tsc",
-    "test": "vitest run",
-    "test:watch": "vitest",
-    "typecheck": "tsc --noEmit"
-  },
   "keywords": [
     "graphql",
     "atproto",
@@ -27,14 +21,20 @@
   "license": "MIT",
   "peerDependencies": {
     "better-sqlite3": ">=11.0.0",
-    "lex-gql": ">=0.2.0"
+    "lex-gql": ">=0.2.1"
   },
   "devDependencies": {
     "@types/better-sqlite3": "^7.6.12",
     "@types/node": "^22.10.0",
     "better-sqlite3": "^12.6.0",
-    "lex-gql": "workspace:*",
     "typescript": "^5.7.2",
-    "vitest": "^1.6.1"
+    "vitest": "^1.6.1",
+    "lex-gql": "0.2.1"
+  },
+  "scripts": {
+    "build": "tsc",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "typecheck": "tsc --noEmit"
   }
-}
+}

package/src/lex-gql-sqlite.js

@@ -2,7 +2,7 @@
  * SQLite adapter for lex-gql
  */
 
-import { hydrateRecord } from 'lex-gql';
+import { hydrateRecord, DEFAULT_SORT } from 'lex-gql';
 
 /**
  * SQL schema for lex-gql records
@@ -110,6 +110,118 @@ const SYSTEM_FIELDS = {
   actorHandle: 'a.handle',
 };
 
+/**
+ * Decode a sort-field-aware cursor
+ * @param {string} cursor - Base64 encoded cursor
+ * @param {Array<{field: string, dir?: string}>} sortFields - Sort configuration
+ * @returns {{ fieldValues: any[], uri: string } | null}
+ */
+function decodeCursor(cursor, sortFields) {
+  try {
+    const decoded = Buffer.from(cursor, 'base64').toString();
+    const parsed = JSON.parse(decoded);
+
+    // Validate cursor structure
+    if (!parsed.v || !Array.isArray(parsed.v) || !parsed.u) {
+      console.debug('lex-gql-sqlite: Invalid cursor structure');
+      return null;
+    }
+
+    // Validate field count matches sort configuration
+    const expectedCount = sortFields?.length || 1;
+    if (parsed.v.length !== expectedCount) {
+      console.debug(
+        `lex-gql-sqlite: Cursor field count mismatch (got ${parsed.v.length}, expected ${expectedCount})`
+      );
+      return null;
+    }
+
+    return { fieldValues: parsed.v, uri: parsed.u };
+  } catch (err) {
+    console.debug('lex-gql-sqlite: Failed to decode cursor:', /** @type {Error} */ (err).message);
+    return null;
+  }
+}
+
+/**
+ * Get SQL comparison operator based on sort direction and cursor type
+ * @param {string | undefined} direction - Sort direction ('asc' or 'desc')
+ * @param {boolean} isBefore - Whether this is a 'before' cursor
+ * @returns {string}
+ */
+function getComparisonOp(direction, isBefore) {
+  const isDesc = direction?.toLowerCase() === 'desc';
+  return isBefore ? (isDesc ? '>' : '<') : (isDesc ? '<' : '>');
+}
+
+/**
+ * Get SQL expression for a field, handling system vs record fields
+ * @param {string} field - Field name
+ * @returns {string}
+ */
+function fieldToSqlExpr(field) {
+  // Validate field name to prevent SQL injection (defense-in-depth)
+  if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(field)) {
+    throw new Error(`Invalid field name: ${field}`);
+  }
+  return SYSTEM_FIELDS[field] || `json_extract(r.record, '$.${field}')`;
+}
+
+/**
+ * Build WHERE clause for cursor pagination with sort field awareness
+ * @param {string} cursor - Base64 encoded cursor
+ * @param {Array<{field: string, dir?: string}>} sortFields - Sort configuration
+ * @param {boolean} isBefore - Whether this is a 'before' cursor
+ * @returns {{ sql: string, params: any[] }}
+ */
+function buildCursorWhere(cursor, sortFields, isBefore) {
+  // Default to indexedAt DESC if no sort fields specified
+  const effectiveSort = sortFields && sortFields.length > 0 ? sortFields : DEFAULT_SORT;
+
+  const decoded = decodeCursor(cursor, effectiveSort);
+  if (!decoded || decoded.fieldValues.length === 0) {
+    return { sql: '1=1', params: [] };
+  }
+
+  const clauses = [];
+  const params = [];
+
+  // Build progressive OR clauses for multi-field sort
+  // For sort [A DESC, B ASC], after cursor [a_val, b_val, uri]:
+  // (A < a_val) OR (A = a_val AND B > b_val) OR (A = a_val AND B = b_val AND uri < uri_val)
+  for (let i = 0; i < effectiveSort.length; i++) {
+    const clauseParts = [];
+    const clauseParams = [];
+
+    // Add equality for all prior fields
+    for (let j = 0; j < i; j++) {
+      clauseParts.push(`${fieldToSqlExpr(effectiveSort[j].field)} = ?`);
+      clauseParams.push(decoded.fieldValues[j]);
+    }
+
+    // Add comparison for current field
+    const op = getComparisonOp(effectiveSort[i].dir, isBefore);
+    clauseParts.push(`${fieldToSqlExpr(effectiveSort[i].field)} ${op} ?`);
+    clauseParams.push(decoded.fieldValues[i]);
+
+    clauses.push(`(${clauseParts.join(' AND ')})`);
+    params.push(...clauseParams);
+  }
+
+  // Final clause: all sort fields equal, compare by URI (tiebreaker)
+  const allEqualParts = effectiveSort.map((s) => {
+    return `${fieldToSqlExpr(s.field)} = ?`;
+  });
+  const lastDir = effectiveSort[effectiveSort.length - 1]?.dir || 'desc';
+  const uriOp = getComparisonOp(lastDir, isBefore);
+  allEqualParts.push(`r.uri ${uriOp} ?`);
+
+  clauses.push(`(${allEqualParts.join(' AND ')})`);
+  params.push(...decoded.fieldValues, decoded.uri);
+
+  return { sql: `(${clauses.join(' OR ')})`, params };
+}
+
 /**
  * Build SQL WHERE clause from lex-gql where conditions
  * @param {import('lex-gql').WhereClause[]} where
@@ -204,15 +316,19 @@ export function buildWhere(where) {
  */
 export function buildOrderBy(sort) {
   if (!sort || sort.length === 0) {
-    return 'r.id DESC';
+    return 'r.indexed_at DESC, r.uri DESC';
   }
 
-  return sort
-    .map(({ field, dir = 'asc' }) => {
-      const fieldPath = SYSTEM_FIELDS[field] || `json_extract(r.record, '$.${field}')`;
-      return `${fieldPath} ${dir.toUpperCase()}`;
-    })
-    .join(', ');
+  const sortClauses = sort.map(({ field, dir = 'asc' }) => {
+    const fieldPath = SYSTEM_FIELDS[field] || `json_extract(r.record, '$.${field}')`;
+    return `${fieldPath} ${dir.toUpperCase()}`;
+  });
+
+  // Add URI as tiebreaker using direction of last sort field
+  const lastDir = sort[sort.length - 1]?.dir || 'asc';
+  sortClauses.push(`r.uri ${lastDir.toUpperCase()}`);
+
+  return sortClauses.join(', ');
 }
 
 /**
@@ -251,26 +367,18 @@ function findMany(db, op) {
   const cursorParams = [];
 
   if (after) {
-    try {
-      const cursor = JSON.parse(Buffer.from(after, 'base64').toString());
-      if (cursor.id) {
-        cursorConditions.push('r.id < ?');
-        cursorParams.push(cursor.id);
-      }
-    } catch (err) {
-      console.warn('lex-gql-sqlite: Malformed cursor (after), ignoring:', /** @type {Error} */ (err).message);
+    const cursorWhere = buildCursorWhere(after, sort, false);
+    if (cursorWhere.sql !== '1=1') {
+      cursorConditions.push(cursorWhere.sql);
+      cursorParams.push(...cursorWhere.params);
     }
   }
 
   if (before) {
-    try {
-      const cursor = JSON.parse(Buffer.from(before, 'base64').toString());
-      if (cursor.id) {
-        cursorConditions.push('r.id > ?');
-        cursorParams.push(cursor.id);
-      }
-    } catch (err) {
-      console.warn('lex-gql-sqlite: Malformed cursor (before), ignoring:', /** @type {Error} */ (err).message);
+    const cursorWhere = buildCursorWhere(before, sort, true);
+    if (cursorWhere.sql !== '1=1') {
+      cursorConditions.push(cursorWhere.sql);
+      cursorParams.push(...cursorWhere.params);
     }
   }
 

package/test/lex-gql-sqlite.test.js

@@ -222,32 +222,32 @@ describe('buildWhere', () => {
 describe('buildOrderBy', () => {
   it('returns default order when no sort', () => {
     const sql = buildOrderBy([]);
-    expect(sql).toBe('r.id DESC');
+    expect(sql).toBe('r.indexed_at DESC, r.uri DESC');
   });
 
-  it('handles single sort field', () => {
+  it('handles single sort field with uri tiebreaker', () => {
     const sql = buildOrderBy([{ field: 'createdAt', dir: 'asc' }]);
-    expect(sql).toBe("json_extract(r.record, '$.createdAt') ASC");
+    expect(sql).toBe("json_extract(r.record, '$.createdAt') ASC, r.uri ASC");
   });
 
-  it('handles system field sort', () => {
+  it('handles system field sort with uri tiebreaker', () => {
     const sql = buildOrderBy([{ field: 'indexedAt', dir: 'desc' }]);
-    expect(sql).toBe('r.indexed_at DESC');
+    expect(sql).toBe('r.indexed_at DESC, r.uri DESC');
   });
 
-  it('handles multi-field sort', () => {
+  it('handles multi-field sort with uri tiebreaker', () => {
     const sql = buildOrderBy([
       { field: 'status', dir: 'asc' },
       { field: 'createdAt', dir: 'desc' },
     ]);
     expect(sql).toBe(
-      "json_extract(r.record, '$.status') ASC, json_extract(r.record, '$.createdAt') DESC",
+      "json_extract(r.record, '$.status') ASC, json_extract(r.record, '$.createdAt') DESC, r.uri DESC",
     );
   });
 
   it('defaults to asc when dir not specified', () => {
     const sql = buildOrderBy([{ field: 'name' }]);
-    expect(sql).toBe("json_extract(r.record, '$.name') ASC");
+    expect(sql).toBe("json_extract(r.record, '$.name') ASC, r.uri ASC");
   });
 });
 
@@ -334,7 +334,9 @@ describe('findMany', () => {
       pagination: { first: 2 },
     });
 
-    const cursor = Buffer.from(JSON.stringify({ id: first.rows[1]._id })).toString('base64');
+    // Cursor format: JSON { v: [sortValues], u: uri } (default sort is indexedAt DESC)
+    const lastRow = first.rows[1];
+    const cursor = Buffer.from(JSON.stringify({ v: [lastRow.indexedAt], u: lastRow.uri })).toString('base64');
 
     const second = await query({
       type: 'findMany',
@@ -347,6 +349,216 @@ describe('findMany', () => {
     expect(second.hasPrev).toBe(true);
   });
 
+  it('handles cursor pagination with custom sort field', async () => {
+    // Insert records with different playedTime values (out of insertion order)
+    writer.insertRecord({
+      uri: 'at://did:plc:abc/col/1',
+      record: { playedTime: '2024-01-03T00:00:00Z', name: 'third' },
+    });
+    writer.insertRecord({
+      uri: 'at://did:plc:abc/col/2',
+      record: { playedTime: '2024-01-01T00:00:00Z', name: 'first' },
+    });
+    writer.insertRecord({
+      uri: 'at://did:plc:abc/col/3',
+      record: { playedTime: '2024-01-05T00:00:00Z', name: 'fifth' },
+    });
+    writer.insertRecord({
+      uri: 'at://did:plc:abc/col/4',
+      record: { playedTime: '2024-01-02T00:00:00Z', name: 'second' },
+    });
+    writer.insertRecord({
+      uri: 'at://did:plc:abc/col/5',
+      record: { playedTime: '2024-01-04T00:00:00Z', name: 'fourth' },
+    });
+
+    // Get first page sorted by playedTime DESC
+    const first = await query({
+      type: 'findMany',
+      collection: 'col',
+      where: [],
+      sort: [{ field: 'playedTime', dir: 'desc' }],
+      pagination: { first: 2 },
+    });
+
+    expect(first.rows).toHaveLength(2);
+    expect(first.rows[0].name).toBe('fifth');  // 2024-01-05
+    expect(first.rows[1].name).toBe('fourth'); // 2024-01-04
+    expect(first.hasNext).toBe(true);
+
+    // Build cursor from last row
+    const lastRow = first.rows[1];
+    const cursor = Buffer.from(JSON.stringify({ v: [lastRow.playedTime], u: lastRow.uri })).toString('base64');
+
+    // Get second page
+    const second = await query({
+      type: 'findMany',
+      collection: 'col',
+      where: [],
+      sort: [{ field: 'playedTime', dir: 'desc' }],
+      pagination: { first: 2, after: cursor },
+    });
+
+    expect(second.rows).toHaveLength(2);
+    expect(second.rows[0].name).toBe('third');  // 2024-01-03
+    expect(second.rows[1].name).toBe('second'); // 2024-01-02
+    expect(second.hasNext).toBe(true);
+
+    // Get third page
+    const lastRow2 = second.rows[1];
+    const cursor2 = Buffer.from(JSON.stringify({ v: [lastRow2.playedTime], u: lastRow2.uri })).toString('base64');
+
+    const third = await query({
+      type: 'findMany',
+      collection: 'col',
+      where: [],
+      sort: [{ field: 'playedTime', dir: 'desc' }],
+      pagination: { first: 2, after: cursor2 },
+    });
+
+    expect(third.rows).toHaveLength(1);
+    expect(third.rows[0].name).toBe('first'); // 2024-01-01
+    expect(third.hasNext).toBe(false);
+  });
+
+  it('does not produce duplicate records when paginating with custom sort', async () => {
+    // Insert 10 records with varying timestamps
+    const timestamps = [
+      '2024-01-10T00:00:00Z',
+      '2024-01-09T00:00:00Z',
+      '2024-01-08T00:00:00Z',
+      '2024-01-07T00:00:00Z',
+      '2024-01-06T00:00:00Z',
+      '2024-01-05T00:00:00Z',
+      '2024-01-04T00:00:00Z',
+      '2024-01-03T00:00:00Z',
+      '2024-01-02T00:00:00Z',
+      '2024-01-01T00:00:00Z',
+    ];
+
+    // Insert in random order
+    const insertOrder = [4, 7, 1, 9, 2, 5, 0, 8, 3, 6];
+    for (const i of insertOrder) {
+      writer.insertRecord({
+        uri: `at://did:plc:abc/col/${i}`,
+        record: { playedTime: timestamps[i], index: i },
+      });
+    }
+
+    const allUris = new Set();
+    let cursor = null;
+
+    // Paginate through all records
+    for (let page = 0; page < 5; page++) {
+      const result = await query({
+        type: 'findMany',
+        collection: 'col',
+        where: [],
+        sort: [{ field: 'playedTime', dir: 'desc' }],
+        pagination: { first: 3, after: cursor },
+      });
+
+      // Check for duplicates
+      for (const row of result.rows) {
+        expect(allUris.has(row.uri)).toBe(false);
+        allUris.add(row.uri);
+      }
+
+      if (!result.hasNext) break;
+
+      // Build cursor for next page
+      const lastRow = result.rows[result.rows.length - 1];
+      cursor = Buffer.from(JSON.stringify({ v: [lastRow.playedTime], u: lastRow.uri })).toString('base64');
+    }
+
+    // Verify we got all records
+    expect(allUris.size).toBe(10);
+  });
+
+  it('handles before cursor with custom sort field', async () => {
+    // Insert records
+    for (let i = 1; i <= 5; i++) {
+      writer.insertRecord({
+        uri: `at://did:plc:abc/col/${i}`,
+        record: { playedTime: `2024-01-0${i}T00:00:00Z`, name: `item${i}` },
+      });
+    }
+
+    // Start from the middle: item3 (2024-01-03)
+    const cursor = Buffer.from(JSON.stringify({ v: ['2024-01-03T00:00:00Z'], u: 'at://did:plc:abc/col/3' })).toString('base64');
+
+    // Get records BEFORE item3 (should be item4, item5 in DESC order)
+    const result = await query({
+      type: 'findMany',
+      collection: 'col',
+      where: [],
+      sort: [{ field: 'playedTime', dir: 'desc' }],
+      pagination: { last: 2, before: cursor },
+    });
+
+    expect(result.rows).toHaveLength(2);
+    expect(result.rows[0].name).toBe('item5'); // 2024-01-05
+    expect(result.rows[1].name).toBe('item4'); // 2024-01-04
+  });
+
+  it('handles cursor pagination with multi-field sort', async () => {
+    // Insert records with two sort fields: category and playedTime
+    const records = [
+      { uri: 'at://did:plc:abc/col/1', category: 'A', playedTime: '2024-01-03T00:00:00Z' },
+      { uri: 'at://did:plc:abc/col/2', category: 'A', playedTime: '2024-01-01T00:00:00Z' },
+      { uri: 'at://did:plc:abc/col/3', category: 'B', playedTime: '2024-01-05T00:00:00Z' },
+      { uri: 'at://did:plc:abc/col/4', category: 'A', playedTime: '2024-01-02T00:00:00Z' },
+      { uri: 'at://did:plc:abc/col/5', category: 'B', playedTime: '2024-01-04T00:00:00Z' },
+      { uri: 'at://did:plc:abc/col/6', category: 'C', playedTime: '2024-01-01T00:00:00Z' },
+    ];
+
+    for (const r of records) {
+      writer.insertRecord({
+        uri: r.uri,
+        record: { category: r.category, playedTime: r.playedTime },
+      });
+    }
+
+    // Sort by category ASC, then playedTime DESC
+    // Expected order: A(01-03), A(01-02), A(01-01), B(01-05), B(01-04), C(01-01)
+    const allUris = [];
+    let cursor = null;
+
+    // Paginate through all records with page size 2
+    for (let page = 0; page < 4; page++) {
+      const result = await query({
+        type: 'findMany',
+        collection: 'col',
+        where: [],
+        sort: [{ field: 'category', dir: 'asc' }, { field: 'playedTime', dir: 'desc' }],
+        pagination: { first: 2, after: cursor },
+      });
+
+      for (const row of result.rows) {
+        allUris.push(row.uri);
+      }
+
+      if (!result.hasNext) break;
+
+      // Build cursor with both sort field values
+      const lastRow = result.rows[result.rows.length - 1];
+      cursor = Buffer.from(JSON.stringify({
+        v: [lastRow.category, lastRow.playedTime],
+        u: lastRow.uri,
+      })).toString('base64');
+    }
+
+    // Verify correct order
+    expect(allUris).toEqual([
+      'at://did:plc:abc/col/1', // A, 01-03
+      'at://did:plc:abc/col/4', // A, 01-02
+      'at://did:plc:abc/col/2', // A, 01-01
+      'at://did:plc:abc/col/3', // B, 01-05
+      'at://did:plc:abc/col/5', // B, 01-04
+      'at://did:plc:abc/col/6', // C, 01-01
+    ]);
+  });
+
   it('filters with where clause', async () => {
     writer.insertRecord({
       uri: 'at://did:plc:abc/col/1',