@hatk/hatk 0.0.1-alpha.84 → 0.0.1-alpha.86

package/dist/database/db.js

@@ -3,6 +3,8 @@ import { getSearchColumns, stripStopWords, getSearchPort, updateFtsRecord, delet
 import { emit, timer } from "../logger.js";
 import { OAUTH_DDL } from "../oauth/db.js";
 import { getDialect } from "./dialect.js";
+import { collectionFromRecordUri, spaceFromUri } from "../spaces/uri.js";
+import { spaceFilterSql } from "../spaces/visibility.js";
 let port;
 let dialect;
 const schemas = new Map();
@@ -80,6 +82,33 @@ export async function initDatabase(adapter, dbPath, tableSchemas, ddlStatements)
     key TEXT PRIMARY KEY,
     value TEXT NOT NULL
   )`);
+    // Permissioned-space sync state. `_space_watch` is one row per space this
+    // instance follows; `_space_repos` is the writer set, which is the sync
+    // boundary — a space is the aggregation of one repo per writer, each on that
+    // writer's own PDS, so progress is tracked per (space, writer) rather than
+    // per space.
+    //
+    // `reader_did` is the account whose session last minted a credential for the
+    // space. hatk is not itself a member of anything: it reads as somebody who
+    // is, so which somebody worked last is worth remembering.
+    await run(`CREATE TABLE IF NOT EXISTS _space_watch (
+    space TEXT PRIMARY KEY,
+    authority TEXT NOT NULL,
+    space_type TEXT NOT NULL,
+    reader_did TEXT,
+    registered_until TEXT,
+    last_error TEXT,
+    updated_at TEXT NOT NULL
+  )`);
+    await run(`CREATE TABLE IF NOT EXISTS _space_repos (
+    space TEXT NOT NULL,
+    did TEXT NOT NULL,
+    pds TEXT,
+    rev TEXT,
+    synced_at TEXT,
+    PRIMARY KEY (space, did)
+  )`);
+    await run(`CREATE INDEX IF NOT EXISTS idx_space_repos_space ON _space_repos(space)`);
     // Labels table (atproto-compatible)
     if (dialect.supportsSequences) {
         await run(`CREATE SEQUENCE IF NOT EXISTS _labels_seq START 1`);
@@ -255,6 +284,7 @@ export async function migrateSchema(tableSchemas) {
         expectedCols.set('uri', 'TEXT');
         expectedCols.set('cid', 'TEXT');
         expectedCols.set('did', 'TEXT');
+        expectedCols.set('space', 'TEXT');
         expectedCols.set('indexed_at', normalizeType(dialect.timestampType));
         for (const col of schema.columns) {
             expectedCols.set(col.name, normalizeType(col.sqlType));
@@ -328,24 +358,37 @@ async function applyMigrationChanges(changes) {
     for (const change of changes) {
         const quotedTable = `"${change.table}"`;
         const quotedColumn = `"${change.column}"`;
+        // Index naming mirrors generateCreateTableSQL and the 'add' branch below:
+        // the table name with dots replaced by underscores, then the column name.
+        const indexName = `idx_${change.table.replace(/\./g, '_')}_${change.column}`;
+        // SQLite refuses to drop a column an index still references ("error in
+        // index ... after drop column"), so any DROP COLUMN has to clear it first.
+        const dropColumn = async () => {
+            await run(`DROP INDEX IF EXISTS "${indexName}"`);
+            await run(`ALTER TABLE ${quotedTable} DROP COLUMN ${quotedColumn}`);
+        };
         try {
             switch (change.action) {
                 case 'add': {
                     await run(`ALTER TABLE ${quotedTable} ADD COLUMN ${quotedColumn} ${change.type}`);
                     emit('migration', 'add_column', { table: change.table, column: change.column, type: change.type });
                     const schema = schemas.get(change.table);
-                    if (schema?.refColumns.includes(change.column)) {
-                        const prefix = change.table.replace(/\./g, '_');
-                        await run(`CREATE INDEX IF NOT EXISTS idx_${prefix}_${change.column} ON ${quotedTable}(${quotedColumn})`);
+                    // `space` is indexed like a ref column but is not one: it is in the
+                    // base row rather than the lexicon, so refColumns never names it, and
+                    // an existing table that gains it would otherwise scan on every read.
+                    if (change.column === 'space' || schema?.refColumns.includes(change.column)) {
+                        await run(`CREATE INDEX IF NOT EXISTS ${indexName} ON ${quotedTable}(${quotedColumn})`);
                     }
                     break;
                 }
                 case 'drop':
-                    await run(`ALTER TABLE ${quotedTable} DROP COLUMN ${quotedColumn}`);
+                    await dropColumn();
                     emit('migration', 'drop_column', { table: change.table, column: change.column });
                     break;
                 case 'retype':
-                    await run(`ALTER TABLE ${quotedTable} DROP COLUMN ${quotedColumn}`);
+                    // A ref column is always TEXT, so a type change means it stopped
+                    // being one — the index goes with the old column and is not rebuilt.
+                    await dropColumn();
                     await run(`ALTER TABLE ${quotedTable} ADD COLUMN ${quotedColumn} ${change.type}`);
                     emit('migration', 'retype_column', { table: change.table, column: change.column, type: change.type });
                     break;
@@ -502,7 +545,8 @@ export async function getRecentRecords(collection, limit) {
     const schema = schemas.get(collection);
     if (!schema)
         return [];
-    const rows = await all(`SELECT t.* FROM ${schema.tableName} t JOIN _repos r ON t.did = r.did WHERE t.indexed_at > r.backfilled_at ORDER BY t.indexed_at DESC LIMIT $1`, [limit]);
+    const gate = spaceFilterSql('t', 2);
+    const rows = await all(`SELECT t.* FROM ${schema.tableName} t JOIN _repos r ON t.did = r.did WHERE t.indexed_at > r.backfilled_at AND ${gate.sql} ORDER BY t.indexed_at DESC LIMIT $1`, [limit, ...gate.params]);
     return rows;
 }
 export async function getSchemaDump() {
@@ -544,10 +588,13 @@ export function buildInsertOp(collection, uri, cid, authorDid, record) {
     const schema = schemas.get(collection);
     if (!schema)
         throw new Error(`Unknown collection: ${collection}`);
-    const colNames = ['uri', 'cid', 'did', 'indexed_at'];
-    const placeholders = ['$1', '$2', '$3', '$4'];
-    const values = [uri, cid, authorDid, new Date().toISOString()];
-    let paramIdx = 5;
+    // Derived from the URI rather than passed in. A space record's URI already
+    // names its space, so reading it here means no write path can land a space
+    // row with the column left NULL — which would publish it to every reader.
+    const colNames = ['uri', 'cid', 'did', 'space', 'indexed_at'];
+    const placeholders = ['$1', '$2', '$3', '$4', '$5'];
+    const values = [uri, cid, authorDid, spaceFromUri(uri) ?? null, new Date().toISOString()];
+    let paramIdx = 6;
     for (const col of schema.columns) {
         let rawValue = record[col.originalName];
         // Handle strongRef expansion: subject_uri reads record.subject.uri, subject__cid reads record.subject.cid
@@ -698,6 +745,57 @@ export async function deleteRecord(collection, uri) {
     }
     await run(`DELETE FROM ${schema.tableName} WHERE uri = $1`, [uri]);
 }
+/**
+ * Remove every row a writer holds in one space.
+ *
+ * Scoped by `(space, did)` rather than by DID alone: the same account can hold
+ * public repo records in the same collections, and a full re-read of one space
+ * must not take those with it. Child and union rows are cleared by parent URI
+ * for the same reason — `parent_did` does not say which space a row came from.
+ */
+export async function purgeSpaceRecords(space, did, collections) {
+    const CHUNK = 500;
+    for (const collection of collections) {
+        const schema = schemas.get(collection);
+        if (!schema)
+            continue;
+        const rows = await all(`SELECT uri FROM ${schema.tableName} WHERE space = $1 AND did = $2`, [
+            space,
+            did,
+        ]);
+        const uris = rows.map((r) => r.uri);
+        if (uris.length === 0)
+            continue;
+        for (const uri of uris)
+            await deleteFtsRecord(collection, uri);
+        for (let i = 0; i < uris.length; i += CHUNK) {
+            const batch = uris.slice(i, i + CHUNK);
+            const placeholders = batch.map((_, j) => `$${j + 1}`).join(',');
+            for (const child of schema.children) {
+                await run(`DELETE FROM ${child.tableName} WHERE parent_uri IN (${placeholders})`, batch);
+            }
+            for (const union of schema.unions) {
+                for (const branch of union.branches) {
+                    await run(`DELETE FROM ${branch.tableName} WHERE parent_uri IN (${placeholders})`, batch);
+                }
+            }
+        }
+        await run(`DELETE FROM ${schema.tableName} WHERE space = $1 AND did = $2`, [space, did]);
+    }
+}
+/** Every writer holding rows for a space, as the index currently has it. */
+export async function listSpaceRecordDids(space, collections) {
+    const dids = new Set();
+    for (const collection of collections) {
+        const schema = schemas.get(collection);
+        if (!schema)
+            continue;
+        const rows = await all(`SELECT DISTINCT did FROM ${schema.tableName} WHERE space = $1`, [space]);
+        for (const row of rows)
+            dids.add(row.did);
+    }
+    return [...dids];
+}
 export async function insertLabels(labels) {
     if (labels.length === 0)
         return;
@@ -737,6 +835,12 @@ export async function queryLabelsForUris(uris) {
     }
     return result;
 }
+/** Per-process token + counter, so concurrent calls never share a staging table. */
+const STAGING_TOKEN = Math.floor(Math.random() * 0xffffff).toString(36);
+let stagingCounter = 0;
+function nextStagingId() {
+    return `${STAGING_TOKEN}_${(stagingCounter++).toString(36)}`;
+}
 export async function bulkInsertRecords(records) {
     if (records.length === 0)
         return 0;
@@ -752,171 +856,192 @@ export async function bulkInsertRecords(records) {
         const schema = schemas.get(collection);
         if (!schema)
             continue;
-        const stagingTable = `_staging_${collection.replace(/\./g, '_')}`;
-        const allCols = ['uri', 'cid', 'did', 'indexed_at', ...schema.columns.map((c) => q(c.name))];
-        const colDefs = [
-            'uri TEXT',
-            'cid TEXT',
-            'did TEXT',
-            'indexed_at TEXT',
-            ...schema.columns.map((c) => {
-                const t = c.sqlType;
-                // Use TEXT for timestamp columns in staging (will cast on merge)
-                return `${q(c.name)} ${t === 'TIMESTAMP' || t === 'TIMESTAMPTZ' ? 'TEXT' : t}`;
-            }),
-        ];
-        await port.execute(`DROP TABLE IF EXISTS ${stagingTable}`, []);
-        await port.execute(`CREATE TABLE ${stagingTable} (${colDefs.join(', ')})`, []);
-        const inserter = await port.createBulkInserter(stagingTable, allCols);
-        const now = new Date().toISOString();
-        for (const rec of recs) {
-            try {
-                const values = [rec.uri, rec.cid, rec.did, now];
-                for (const col of schema.columns) {
-                    values.push(resolveColumnValue(col, rec.record));
-                }
-                inserter.append(values);
-                inserted++;
-            }
-            catch {
-                // Skip bad records
-            }
-        }
-        await inserter.close();
-        // Merge into target, filtering rows that would violate NOT NULL
-        const selectCols = allCols.map((name) => {
-            const col = schema.columns.find((c) => q(c.name) === name);
-            if (name === 'indexed_at' || (col && (col.sqlType === 'TIMESTAMP' || col.sqlType === 'TIMESTAMPTZ'))) {
-                return `${dialect.tryCastTimestamp(name)} AS ${name}`;
-            }
+        // Staging tables are named per call, not per collection: runBackfill
+        // imports repos in parallel, so two workers on the same collection would
+        // otherwise drop and recreate each other's staging table mid-insert.
+        const stagingTables = [];
+        const stagingName = (suffix) => {
+            const name = `_staging_${collection.replace(/\./g, '_')}${suffix}_${nextStagingId()}`;
+            stagingTables.push(name);
             return name;
-        });
-        const notNullChecks = ['uri IS NOT NULL', 'did IS NOT NULL'];
-        for (const col of schema.columns) {
-            if (col.notNull) {
-                if (col.sqlType === 'TIMESTAMP' || col.sqlType === 'TIMESTAMPTZ') {
-                    notNullChecks.push(`${dialect.tryCastTimestamp(q(col.name))} IS NOT NULL`);
-                }
-                else {
-                    notNullChecks.push(`${q(col.name)} IS NOT NULL`);
-                }
-            }
-        }
-        const whereClause = notNullChecks.length ? ` WHERE ${notNullChecks.join(' AND ')}` : '';
-        await port.execute(`INSERT OR REPLACE INTO ${schema.tableName} (${allCols.join(', ')}) SELECT ${selectCols.join(', ')} FROM ${stagingTable}${whereClause}`, []);
-        await port.execute(`DROP TABLE ${stagingTable}`, []);
-        // Populate child tables
-        for (const child of schema.children) {
-            const childStagingTable = `_staging_${collection.replace(/\./g, '_')}__${child.fieldName}`;
-            const childColDefs = [
-                'parent_uri TEXT',
-                'parent_did TEXT',
-                ...child.columns.map((c) => {
+        };
+        try {
+            const stagingTable = stagingName('');
+            const allCols = ['uri', 'cid', 'did', 'space', 'indexed_at', ...schema.columns.map((c) => q(c.name))];
+            const colDefs = [
+                'uri TEXT',
+                'cid TEXT',
+                'did TEXT',
+                'space TEXT',
+                'indexed_at TEXT',
+                ...schema.columns.map((c) => {
                     const t = c.sqlType;
+                    // Use TEXT for timestamp columns in staging (will cast on merge)
                     return `${q(c.name)} ${t === 'TIMESTAMP' || t === 'TIMESTAMPTZ' ? 'TEXT' : t}`;
                 }),
             ];
-            const childAllCols = ['parent_uri', 'parent_did', ...child.columns.map((c) => q(c.name))];
-            await port.execute(`DROP TABLE IF EXISTS ${childStagingTable}`, []);
-            await port.execute(`CREATE TABLE ${childStagingTable} (${childColDefs.join(', ')})`, []);
-            const childInserter = await port.createBulkInserter(childStagingTable, childAllCols);
+            await port.execute(`DROP TABLE IF EXISTS ${stagingTable}`, []);
+            await port.execute(`CREATE TABLE ${stagingTable} (${colDefs.join(', ')})`, []);
+            const inserter = await port.createBulkInserter(stagingTable, allCols);
+            const now = new Date().toISOString();
             for (const rec of recs) {
-                const items = rec.record[child.fieldName];
-                if (!Array.isArray(items))
-                    continue;
-                for (const item of items) {
-                    try {
-                        const values = [rec.uri, rec.did];
-                        for (const col of child.columns) {
-                            values.push(resolveRawColumnValue(col, item));
-                        }
-                        childInserter.append(values);
-                    }
-                    catch {
-                        // Skip bad items
+                try {
+                    const values = [rec.uri, rec.cid, rec.did, spaceFromUri(rec.uri) ?? null, now];
+                    for (const col of schema.columns) {
+                        values.push(resolveColumnValue(col, rec.record));
                     }
+                    inserter.append(values);
+                    inserted++;
+                }
+                catch {
+                    // Skip bad records
                 }
             }
-            await childInserter.close();
-            // Delete existing child rows for these URIs, then merge staging
-            const uriPlaceholders = recs.map((_, i) => `$${i + 1}`).join(',');
-            await port.execute(`DELETE FROM ${child.tableName} WHERE parent_uri IN (${uriPlaceholders})`, recs.map((r) => r.uri));
-            const childSelectCols = childAllCols.map((name) => {
-                const col = child.columns.find((c) => q(c.name) === name);
-                if (col && (col.sqlType === 'TIMESTAMP' || col.sqlType === 'TIMESTAMPTZ')) {
+            await inserter.close();
+            // Merge into target, filtering rows that would violate NOT NULL
+            const selectCols = allCols.map((name) => {
+                const col = schema.columns.find((c) => q(c.name) === name);
+                if (name === 'indexed_at' || (col && (col.sqlType === 'TIMESTAMP' || col.sqlType === 'TIMESTAMPTZ'))) {
                     return `${dialect.tryCastTimestamp(name)} AS ${name}`;
                 }
                 return name;
             });
-            await port.execute(`INSERT INTO ${child.tableName} (${childAllCols.join(', ')}) SELECT ${childSelectCols.join(', ')} FROM ${childStagingTable} WHERE parent_uri IS NOT NULL`, []);
-            await port.execute(`DROP TABLE ${childStagingTable}`, []);
-        }
-        // Populate union branch tables
-        for (const union of schema.unions) {
-            for (const branch of union.branches) {
-                const branchStagingTable = `_staging_${collection.replace(/\./g, '_')}__${toSnakeCase(union.fieldName)}_${branch.branchName}`;
-                const branchColDefs = [
+            const notNullChecks = ['uri IS NOT NULL', 'did IS NOT NULL'];
+            for (const col of schema.columns) {
+                if (col.notNull) {
+                    if (col.sqlType === 'TIMESTAMP' || col.sqlType === 'TIMESTAMPTZ') {
+                        notNullChecks.push(`${dialect.tryCastTimestamp(q(col.name))} IS NOT NULL`);
+                    }
+                    else {
+                        notNullChecks.push(`${q(col.name)} IS NOT NULL`);
+                    }
+                }
+            }
+            const whereClause = notNullChecks.length ? ` WHERE ${notNullChecks.join(' AND ')}` : '';
+            await port.execute(`INSERT OR REPLACE INTO ${schema.tableName} (${allCols.join(', ')}) SELECT ${selectCols.join(', ')} FROM ${stagingTable}${whereClause}`, []);
+            await port.execute(`DROP TABLE ${stagingTable}`, []);
+            // Populate child tables
+            for (const child of schema.children) {
+                const childStagingTable = stagingName(`__${child.fieldName}`);
+                const childColDefs = [
                     'parent_uri TEXT',
                     'parent_did TEXT',
-                    ...branch.columns.map((c) => {
+                    ...child.columns.map((c) => {
                         const t = c.sqlType;
                         return `${q(c.name)} ${t === 'TIMESTAMP' || t === 'TIMESTAMPTZ' ? 'TEXT' : t}`;
                     }),
                 ];
-                const branchAllCols = ['parent_uri', 'parent_did', ...branch.columns.map((c) => q(c.name))];
-                await port.execute(`DROP TABLE IF EXISTS ${branchStagingTable}`, []);
-                await port.execute(`CREATE TABLE ${branchStagingTable} (${branchColDefs.join(', ')})`, []);
-                const branchInserter = await port.createBulkInserter(branchStagingTable, branchAllCols);
+                const childAllCols = ['parent_uri', 'parent_did', ...child.columns.map((c) => q(c.name))];
+                await port.execute(`DROP TABLE IF EXISTS ${childStagingTable}`, []);
+                await port.execute(`CREATE TABLE ${childStagingTable} (${childColDefs.join(', ')})`, []);
+                const childInserter = await port.createBulkInserter(childStagingTable, childAllCols);
                 for (const rec of recs) {
-                    const unionValue = rec.record[union.fieldName];
-                    if (!unionValue || typeof unionValue !== 'object')
-                        continue;
-                    if (unionValue.$type !== branch.type)
+                    const items = rec.record[child.fieldName];
+                    if (!Array.isArray(items))
                         continue;
-                    if (branch.isArray && branch.arrayField) {
-                        const items = resolveBranchData(unionValue, branch)[branch.arrayField];
-                        if (!Array.isArray(items))
-                            continue;
-                        for (const item of items) {
-                            try {
-                                const values = [rec.uri, rec.did];
-                                for (const col of branch.columns) {
-                                    values.push(resolveRawColumnValue(col, item));
-                                }
-                                branchInserter.append(values);
-                            }
-                            catch {
-                                // Skip bad items
-                            }
-                        }
-                    }
-                    else {
+                    for (const item of items) {
                         try {
-                            const branchData = resolveBranchData(unionValue, branch);
                             const values = [rec.uri, rec.did];
-                            for (const col of branch.columns) {
-                                values.push(resolveRawColumnValue(col, branchData));
+                            for (const col of child.columns) {
+                                values.push(resolveRawColumnValue(col, item));
                             }
-                            branchInserter.append(values);
+                            childInserter.append(values);
                         }
                         catch {
-                            // Skip bad records
+                            // Skip bad items
                         }
                     }
                 }
-                await branchInserter.close();
-                // Delete existing branch rows for these URIs, then merge staging
+                await childInserter.close();
+                // Delete existing child rows for these URIs, then merge staging
                 const uriPlaceholders = recs.map((_, i) => `$${i + 1}`).join(',');
-                await port.execute(`DELETE FROM ${branch.tableName} WHERE parent_uri IN (${uriPlaceholders})`, recs.map((r) => r.uri));
-                const branchSelectCols = branchAllCols.map((name) => {
-                    const col = branch.columns.find((c) => q(c.name) === name);
+                await port.execute(`DELETE FROM ${child.tableName} WHERE parent_uri IN (${uriPlaceholders})`, recs.map((r) => r.uri));
+                const childSelectCols = childAllCols.map((name) => {
+                    const col = child.columns.find((c) => q(c.name) === name);
                     if (col && (col.sqlType === 'TIMESTAMP' || col.sqlType === 'TIMESTAMPTZ')) {
                         return `${dialect.tryCastTimestamp(name)} AS ${name}`;
                     }
                     return name;
                 });
-                await port.execute(`INSERT INTO ${branch.tableName} (${branchAllCols.join(', ')}) SELECT ${branchSelectCols.join(', ')} FROM ${branchStagingTable} WHERE parent_uri IS NOT NULL`, []);
-                await port.execute(`DROP TABLE ${branchStagingTable}`, []);
+                await port.execute(`INSERT INTO ${child.tableName} (${childAllCols.join(', ')}) SELECT ${childSelectCols.join(', ')} FROM ${childStagingTable} WHERE parent_uri IS NOT NULL`, []);
+                await port.execute(`DROP TABLE ${childStagingTable}`, []);
+            }
+            // Populate union branch tables
+            for (const union of schema.unions) {
+                for (const branch of union.branches) {
+                    const branchStagingTable = stagingName(`__${toSnakeCase(union.fieldName)}_${branch.branchName}`);
+                    const branchColDefs = [
+                        'parent_uri TEXT',
+                        'parent_did TEXT',
+                        ...branch.columns.map((c) => {
+                            const t = c.sqlType;
+                            return `${q(c.name)} ${t === 'TIMESTAMP' || t === 'TIMESTAMPTZ' ? 'TEXT' : t}`;
+                        }),
+                    ];
+                    const branchAllCols = ['parent_uri', 'parent_did', ...branch.columns.map((c) => q(c.name))];
+                    await port.execute(`DROP TABLE IF EXISTS ${branchStagingTable}`, []);
+                    await port.execute(`CREATE TABLE ${branchStagingTable} (${branchColDefs.join(', ')})`, []);
+                    const branchInserter = await port.createBulkInserter(branchStagingTable, branchAllCols);
+                    for (const rec of recs) {
+                        const unionValue = rec.record[union.fieldName];
+                        if (!unionValue || typeof unionValue !== 'object')
+                            continue;
+                        if (unionValue.$type !== branch.type)
+                            continue;
+                        if (branch.isArray && branch.arrayField) {
+                            const items = resolveBranchData(unionValue, branch)[branch.arrayField];
+                            if (!Array.isArray(items))
+                                continue;
+                            for (const item of items) {
+                                try {
+                                    const values = [rec.uri, rec.did];
+                                    for (const col of branch.columns) {
+                                        values.push(resolveRawColumnValue(col, item));
+                                    }
+                                    branchInserter.append(values);
+                                }
+                                catch {
+                                    // Skip bad items
+                                }
+                            }
+                        }
+                        else {
+                            try {
+                                const branchData = resolveBranchData(unionValue, branch);
+                                const values = [rec.uri, rec.did];
+                                for (const col of branch.columns) {
+                                    values.push(resolveRawColumnValue(col, branchData));
+                                }
+                                branchInserter.append(values);
+                            }
+                            catch {
+                                // Skip bad records
+                            }
+                        }
+                    }
+                    await branchInserter.close();
+                    // Delete existing branch rows for these URIs, then merge staging
+                    const uriPlaceholders = recs.map((_, i) => `$${i + 1}`).join(',');
+                    await port.execute(`DELETE FROM ${branch.tableName} WHERE parent_uri IN (${uriPlaceholders})`, recs.map((r) => r.uri));
+                    const branchSelectCols = branchAllCols.map((name) => {
+                        const col = branch.columns.find((c) => q(c.name) === name);
+                        if (col && (col.sqlType === 'TIMESTAMP' || col.sqlType === 'TIMESTAMPTZ')) {
+                            return `${dialect.tryCastTimestamp(name)} AS ${name}`;
+                        }
+                        return name;
+                    });
+                    await port.execute(`INSERT INTO ${branch.tableName} (${branchAllCols.join(', ')}) SELECT ${branchSelectCols.join(', ')} FROM ${branchStagingTable} WHERE parent_uri IS NOT NULL`, []);
+                    await port.execute(`DROP TABLE ${branchStagingTable}`, []);
+                }
+            }
+        }
+        finally {
+            // Drop on the error path too, so a failed batch leaves nothing behind.
+            for (const name of stagingTables) {
+                try {
+                    await port.execute(`DROP TABLE IF EXISTS ${name}`, []);
+                }
+                catch { }
             }
         }
     }
@@ -961,6 +1086,16 @@ export async function queryRecords(collection, opts = {}) {
     if (!schema)
         throw new Error(`Unknown collection: ${collection}`);
     const { limit = 20, cursor, filters, sort = 'indexed_at', order = 'desc' } = opts;
+    // `order` is interpolated into the ORDER BY clause, not bound as a
+    // parameter, so it has to be an exact match rather than a trusted caller
+    // value — anything else is SQL reaching the engine verbatim.
+    const direction = String(order).toUpperCase();
+    if (direction !== 'ASC' && direction !== 'DESC')
+        throw new Error(`Invalid sort order: ${order}`);
+    // SQLite (and DuckDB) read a negative LIMIT as unbounded, so an unchecked
+    // -1 returns the entire table.
+    if (!Number.isInteger(limit) || limit < 1)
+        throw new Error(`Invalid limit: ${limit}`);
     // Validate sort field exists
     const sortCol = sort === 'indexed_at' ? 'indexed_at' : schema.columns.find((c) => c.originalName === sort || c.name === sort);
     const sortName = typeof sortCol === 'string' ? sortCol : sortCol?.name;
@@ -973,7 +1108,7 @@ export async function queryRecords(collection, opts = {}) {
     if (cursor) {
         const parsed = unpackCursor(cursor);
         if (parsed) {
-            const op = order === 'desc' ? '<' : '>';
+            const op = direction === 'DESC' ? '<' : '>';
             const pSort1 = `$${paramIdx++}`;
             const pSort2 = `$${paramIdx++}`;
             const pCid = `$${paramIdx++}`;
@@ -993,10 +1128,15 @@ export async function queryRecords(collection, opts = {}) {
             params.push(value);
         }
     }
+    // Space rows are served only to a read scoped to that space; see spaces/visibility.ts.
+    const gate = spaceFilterSql('t', paramIdx);
+    conditions.push(gate.sql);
+    params.push(...gate.params);
+    paramIdx = gate.nextIdx;
     let sql = `SELECT t.*, r.handle FROM ${schema.tableName} t LEFT JOIN _repos r ON t.did = r.did WHERE (r.status IS NULL OR r.status != 'takendown')`;
     if (conditions.length)
         sql += ' AND ' + conditions.join(' AND ');
-    sql += ` ORDER BY t.${sortName} ${order.toUpperCase()}, t.cid ${order.toUpperCase()} LIMIT $${paramIdx++}`;
+    sql += ` ORDER BY t.${sortName} ${direction}, t.cid ${direction} LIMIT $${paramIdx++}`;
     params.push(limit + 1); // fetch one extra for cursor
     const rows = await all(sql, params);
     const hasMore = rows.length > limit;
@@ -1038,7 +1178,8 @@ export async function queryRecords(collection, opts = {}) {
 }
 export async function getRecordByUri(uri) {
     for (const [_collection, schema] of schemas) {
-        const rows = await all(`SELECT t.*, r.handle FROM ${schema.tableName} t LEFT JOIN _repos r ON t.did = r.did WHERE t.uri = $1 AND (r.status IS NULL OR r.status != 'takendown')`, [uri]);
+        const gate = spaceFilterSql('t', 2);
+        const rows = await all(`SELECT t.*, r.handle FROM ${schema.tableName} t LEFT JOIN _repos r ON t.did = r.did WHERE t.uri = $1 AND ${gate.sql} AND (r.status IS NULL OR r.status != 'takendown')`, [uri, ...gate.params]);
         if (rows.length > 0) {
             const row = rows[0];
             if (schema.children.length > 0) {
@@ -1075,7 +1216,8 @@ export async function getRecordsByUris(collection, uris) {
     if (!schema)
         return [];
     const placeholders = uris.map((_, i) => `$${i + 1}`).join(',');
-    const rows = await all(`SELECT t.*, r.handle FROM ${schema.tableName} t LEFT JOIN _repos r ON t.did = r.did WHERE t.uri IN (${placeholders}) AND (r.status IS NULL OR r.status != 'takendown')`, uris);
+    const gate = spaceFilterSql('t', uris.length + 1);
+    const rows = await all(`SELECT t.*, r.handle FROM ${schema.tableName} t LEFT JOIN _repos r ON t.did = r.did WHERE t.uri IN (${placeholders}) AND ${gate.sql} AND (r.status IS NULL OR r.status != 'takendown')`, [...uris, ...gate.params]);
     // Batch-fetch child rows for all URIs
     const childData = new Map();
     for (const child of schema.children) {
@@ -1161,10 +1303,12 @@ export async function searchRecords(collection, query, opts = {}) {
                 const scoreMap = new Map(hits.map((h) => [h.uri, h.score]));
                 // Fetch full records for matched URIs
                 const placeholders = uriList.map((_, i) => `$${i + 1}`).join(', ');
+                const gate = spaceFilterSql('m', uriList.length + 1);
                 const rows = await all(`SELECT m.* FROM ${schema.tableName} m
           LEFT JOIN _repos r ON m.did = r.did
           WHERE m.uri IN (${placeholders})
-          AND (r.status IS NULL OR r.status != 'takendown')`, uriList);
+          AND ${gate.sql}
+          AND (r.status IS NULL OR r.status != 'takendown')`, [...uriList, ...gate.params]);
                 // Re-attach scores and sort
                 bm25Results = rows
                     .map((r) => ({ ...r, score: scoreMap.get(r.uri) ?? 0 }))
@@ -1199,13 +1343,19 @@ export async function searchRecords(collection, query, opts = {}) {
             }
         }
         const greatestExpr = dialect.greatest(simExprs);
+        const fuzzyGate = spaceFilterSql('t', 3);
         const fuzzySQL = `SELECT t.*, ${greatestExpr} AS fuzzy_score
       FROM ${schema.tableName} t LEFT JOIN _repos r ON t.did = r.did
       WHERE ${greatestExpr} >= 0.8
+      AND ${fuzzyGate.sql}
       ORDER BY fuzzy_score DESC
       LIMIT $2`;
         try {
-            const fuzzyRows = await all(fuzzySQL, [query, remaining + existingUris.size]);
+            const fuzzyRows = await all(fuzzySQL, [
+                query,
+                remaining + existingUris.size,
+                ...fuzzyGate.params,
+            ]);
             phasesUsed.push('fuzzy');
             for (const row of fuzzyRows) {
                 if (bm25Results.length >= limit)
@@ -1256,9 +1406,8 @@ export async function countByField(collection, field, value) {
     const schema = schemas.get(collection);
     if (!schema)
         return 0;
-    const rows = await all(`SELECT COUNT(*) as count FROM ${schema.tableName} WHERE ${field} = $1`, [
-        value,
-    ]);
+    const gate = spaceFilterSql('', 2);
+    const rows = await all(`SELECT COUNT(*) as count FROM ${schema.tableName} WHERE ${field} = $1 AND ${gate.sql}`, [value, ...gate.params]);
     return Number(rows[0]?.count || 0);
 }
 export async function countByFieldBatch(collection, field, values) {
@@ -1268,7 +1417,8 @@ export async function countByFieldBatch(collection, field, values) {
     if (!schema)
         return new Map();
     const placeholders = values.map((_, i) => `$${i + 1}`).join(',');
-    const rows = await all(`SELECT ${field}, COUNT(*) as count FROM ${schema.tableName} WHERE ${field} IN (${placeholders}) GROUP BY ${field}`, values);
+    const gate = spaceFilterSql('', values.length + 1);
+    const rows = await all(`SELECT ${field}, COUNT(*) as count FROM ${schema.tableName} WHERE ${field} IN (${placeholders}) AND ${gate.sql} GROUP BY ${field}`, [...values, ...gate.params]);
     const result = new Map();
     for (const row of rows) {
         result.set(row[field], Number(row.count));
@@ -1279,7 +1429,11 @@ export async function findByField(collection, field, value) {
     const schema = schemas.get(collection);
     if (!schema)
         return null;
-    const rows = await all(`SELECT * FROM ${schema.tableName} WHERE ${field} = $1 LIMIT 1`, [value]);
+    const gate = spaceFilterSql('', 2);
+    const rows = await all(`SELECT * FROM ${schema.tableName} WHERE ${field} = $1 AND ${gate.sql} LIMIT 1`, [
+        value,
+        ...gate.params,
+    ]);
     return rows[0] || null;
 }
 export async function findByFieldBatch(collection, field, values) {
@@ -1289,7 +1443,8 @@ export async function findByFieldBatch(collection, field, values) {
     if (!schema)
         return new Map();
     const placeholders = values.map((_, i) => `$${i + 1}`).join(',');
-    const rows = await all(`SELECT t.*, r.handle FROM ${schema.tableName} t LEFT JOIN _repos r ON t.did = r.did WHERE t.${field} IN (${placeholders})`, values);
+    const gate = spaceFilterSql('t', values.length + 1);
+    const rows = await all(`SELECT t.*, r.handle FROM ${schema.tableName} t LEFT JOIN _repos r ON t.did = r.did WHERE t.${field} IN (${placeholders}) AND ${gate.sql}`, [...values, ...gate.params]);
     // Attach child data if this collection has decomposed arrays
     if (schema.children.length > 0 && rows.length > 0) {
         const uris = rows.map((r) => r.uri);
@@ -1329,10 +1484,11 @@ export async function findUriByFields(collection, conditions) {
         return null;
     const where = conditions.map((c, i) => `${c.field} = $${i + 1}`).join(' AND ');
     const params = conditions.map((c) => c.value);
-    const rows = await all(`SELECT uri FROM ${schema.tableName} WHERE ${where} LIMIT 1`, params);
+    const gate = spaceFilterSql('', conditions.length + 1);
+    const rows = await all(`SELECT uri FROM ${schema.tableName} WHERE ${where} AND ${gate.sql} LIMIT 1`, [...params, ...gate.params]);
     return rows[0]?.uri || null;
 }
-const ENVELOPE_KEYS = new Set(['uri', 'cid', 'did', 'handle', 'indexed_at']);
+const ENVELOPE_KEYS = new Set(['uri', 'cid', 'did', 'space', 'handle', 'indexed_at']);
 const INTERNAL_KEYS = new Set(['__childData', '__unionData']);
 export function normalizeValue(v) {
     if (v && typeof v === 'object' && 'micros' in v)
@@ -1355,11 +1511,29 @@ export async function getChildRows(childTableName, parentUris) {
     }
     return result;
 }
+/**
+ * Normalize one stored column back to its record value. JSON columns are
+ * written with JSON.stringify on insert, so they have to be parsed on the way
+ * out — on child and union-branch rows just as much as on the main table.
+ */
+function decodeColumn(col, raw) {
+    const val = normalizeValue(raw);
+    if (!col.isJson || typeof val !== 'string')
+        return val;
+    try {
+        return JSON.parse(val);
+    }
+    catch {
+        return val;
+    }
+}
 export function reshapeRow(row, childData, unionData) {
     if (!row)
         return null;
-    // Derive collection from URI (at://did/collection/rkey)
-    const collection = row.uri?.split('/')?.[3];
+    // A space record's URI puts the collection in a different segment than a
+    // repo record's. Reading it positionally found no schema for space rows,
+    // which left their columns snake_cased and their JSON columns unparsed.
+    const collection = row.uri ? collectionFromRecordUri(row.uri) : undefined;
     const schema = collection ? schemas.get(collection) : null;
     // Build snake→camel map and JSON column set from schema
     const nameMap = new Map();
@@ -1396,6 +1570,29 @@ export function reshapeRow(row, childData, unionData) {
             }
         }
     }
+    // A strongRef is stored as two columns so the uri can be joined on, and
+    // reads back as the object the lexicon declares: `{ uri, cid }`. The
+    // generated type for the field says so, and a handler written against it
+    // must find it.
+    if (schema) {
+        for (const col of schema.columns) {
+            if (!col.originalName.endsWith('__cid'))
+                continue;
+            const field = col.originalName.slice(0, -'__cid'.length);
+            const uri = value[field];
+            const cid = value[col.originalName];
+            delete value[col.originalName];
+            if (uri == null && cid == null) {
+                delete value[field];
+                continue;
+            }
+            value[field] = { ...(uri != null ? { uri } : {}), ...(cid != null ? { cid } : {}) };
+        }
+    }
+    // Only space rows carry a space, and an explicit null on every other row
+    // would put a field on the wire that means nothing there.
+    if (envelope.space == null)
+        delete envelope.space;
     // Reconstruct decomposed array fields from child data
     if (schema && childData) {
         for (const child of schema.children) {
@@ -1404,8 +1601,7 @@ export function reshapeRow(row, childData, unionData) {
             value[child.fieldName] = childRows.map((cr) => {
                 const item = {};
                 for (const col of child.columns) {
-                    const raw = cr[col.name];
-                    item[col.originalName] = normalizeValue(raw);
+                    item[col.originalName] = decodeColumn(col, cr[col.name]);
                 }
                 return item;
             });
@@ -1429,7 +1625,7 @@ export function reshapeRow(row, childData, unionData) {
                     const items = branchRows.map((br) => {
                         const item = {};
                         for (const col of branch.columns) {
-                            item[col.originalName] = normalizeValue(br[col.name]);
+                            item[col.originalName] = decodeColumn(col, br[col.name]);
                         }
                         return item;
                     });
@@ -1441,7 +1637,7 @@ export function reshapeRow(row, childData, unionData) {
                     const br = branchRows[0];
                     const props = {};
                     for (const col of branch.columns) {
-                        props[col.originalName] = normalizeValue(br[col.name]);
+                        props[col.originalName] = decodeColumn(col, br[col.name]);
                     }
                     if (branch.wrapperField) {
                         value[union.fieldName] = { $type: branch.type, [branch.wrapperField]: props };