@hatk/hatk 0.0.1-alpha.11 → 0.0.1-alpha.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/db.d.ts +0 -1
- package/dist/db.d.ts.map +1 -1
- package/dist/db.js +4 -39
- package/dist/main.js +1 -3
- package/package.json +1 -1
package/dist/db.d.ts
CHANGED
|
@@ -130,6 +130,5 @@ export declare function isTakendownDid(did: string): Promise<boolean>;
|
|
|
130
130
|
export declare function getPreferences(did: string): Promise<Record<string, any>>;
|
|
131
131
|
export declare function putPreference(did: string, key: string, value: any): Promise<void>;
|
|
132
132
|
export declare function filterTakendownDids(dids: string[]): Promise<Set<string>>;
|
|
133
|
-
export declare function backfillChildTables(): Promise<void>;
|
|
134
133
|
export {};
|
|
135
134
|
//# sourceMappingURL=db.d.ts.map
|
package/dist/db.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../src/db.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,WAAW,EAAe,MAAM,aAAa,CAAA;AAC3D,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,gBAAgB,CAAA;AAUzC,wBAAgB,aAAa,IAAI,IAAI,CAUpC;AA+DD,wBAAsB,QAAQ,CAAC,UAAU,EAAE,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,GAAG,EAAE,CAAA;CAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAkB/F;AAiBD,wBAAsB,YAAY,CAChC,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,WAAW,EAAE,EAC3B,aAAa,EAAE,MAAM,EAAE,GACtB,OAAO,CAAC,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../src/db.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,WAAW,EAAe,MAAM,aAAa,CAAA;AAC3D,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,gBAAgB,CAAA;AAUzC,wBAAgB,aAAa,IAAI,IAAI,CAUpC;AA+DD,wBAAsB,QAAQ,CAAC,UAAU,EAAE,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,GAAG,EAAE,CAAA;CAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAkB/F;AAiBD,wBAAsB,YAAY,CAChC,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,WAAW,EAAE,EAC3B,aAAa,EAAE,MAAM,EAAE,GACtB,OAAO,CAAC,IAAI,CAAC,CAoEf;AAED,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAGnE;AAED,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAEzE;AAED,wBAAsB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAGvE;AAED,wBAAsB,aAAa,CACjC,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,EACd,GAAG,CAAC,EAAE,MAAM,EACZ,IAAI,CAAC,EAAE;IAAE,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAC1E,OAAO,CAAC,IAAI,CAAC,CA0Cf;AAED,wBAAsB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAGpE;AAED,wBAAsB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,CAI9G;AAED,wBAAsB,sBAAsB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAQlF;AAED,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,CAG1D;AAED,wBAAsB,mBAAmB,IAAI,OAAO,CAAC,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC,CAE3F;AAED,wBAAsB,kBAAkB,CACtC,IAAI,GAAE;IACJ,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,CAAC,CAAC,EAAE,MAAM,CAAA;CACN,GACL,OAAO,CAAC;IAAE,KAAK,EAAE,GAAG,EAAE,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CA6B1C;AAED,wBAAsB,mBAAmB,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAO3E;AAED,wBAAsB,aAAa,IAAI,OAAO,CAAC,MAAM,CAAC,CAGrD;AAED,wBAAgB,aAAa,CAC3B,UAAU,EAAE,MAAM,EAClB,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAC1B;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,GAAG,EAAE,CAAA;CAAE,CA+BhC;AAED,wBAAsB,YAAY,CAChC,UAAU,EAAE,MAAM,EAClB,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAwGf;AAWD,wBAAsB,YAAY,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAYjF;AAED,wBAAsB,YAAY,CAChC,MAAM,EAAE,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,OAAO,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,GAClG,OAAO,CAAC,IAAI,CAAC,CAsBf;AAED,wBAAsB,kBAAkB,CACtC,IAAI,EAAE,MAAM,EAAE,GACb,OAAO,CACR,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,OAAO,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,CAAC,CAC7G,CAqBA;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,EAAE,MAAM,CAAA;IAClB,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;CAC5B;AAED,wBAAsB,iBAAiB,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAwQ9E;AAED,UAAU,SAAS;IACjB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAChC,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAA;CACvB;AAED,wBAAsB,YAAY,CAChC,UAAU,EAAE,MAAM,EAClB,IAAI,GAAE,SAAc,GACnB,OAAO,CAAC;IAAE,OAAO,EAAE,GAAG,EAAE,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,CAoF9C;AAED,wBAAsB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAgCrE;AAED,wBAAsB,gBAAgB,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAqCzF;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,aAAa,CACjC,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,EACb,IAAI,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,OAAO,CAAA;CAAO,GAC9D,OAAO,CAAC;IAAE,OAAO,EAAE,GAAG,EAAE,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,CAmN9C;AAGD,wBAAsB,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,GAAE,GAAG,EAAO,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAE9E;AAED,wBAAsB,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAEzE;AAED,wBAAgB,SAAS,CAAC,UAAU,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS,CAErE;AAED,wBAAsB,YAAY,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAKpG;AAED,wBAAsB,iBAAiB,CACrC,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EAAE,GACf,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAc9B;AAED,wBAAsB,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAKvG;AAED,wBAAsB,gBAAgB,CACpC,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EAAE,GACf,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CA6B7B;AAED,wBAAsB,kBAAkB,CACtC,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EAAE,GACf,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CASpC;AAED,wBAAsB,eAAe,CACnC,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,EAAE,GAC7C,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAOxB;AAKD,wBAAgB,cAAc,CAAC,CAAC,EAAE,GAAG,GAAG,GAAG,CAI1C;AAED,wBAAsB,YAAY,CAAC,cAAc,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CAW5G;AAED,wBAAgB,UAAU,CACxB,GAAG,EAAE,GAAG,EACR,SAAS,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,EAC3C,SAAS,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,GACvD,GAAG,CAAC,OAAO,CAAC,GAAG,IAAI,CAiGrB;AAED,wBAAgB,UAAU,CAAC,OAAO,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAGhE;AAED,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CASpF;AAED,wBAAsB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAKlE;AAED,wBAAsB,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,GAAE,MAAW,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAMtF;AAED,wBAAsB,qBAAqB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAOxE;AAED,wBAAsB,sBAAsB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAO3E;AAED,wBAAsB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAGlE;AAED,wBAAsB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAW9E;AAED,wBAAsB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAQvF;AAED,wBAAsB,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAK9E"}
|
package/dist/db.js
CHANGED
|
@@ -114,7 +114,10 @@ async function all(sql, ...params) {
|
|
|
114
114
|
return enqueue('read', () => allDirect(sql, ...params));
|
|
115
115
|
}
|
|
116
116
|
export async function initDatabase(dbPath, tableSchemas, ddlStatements) {
|
|
117
|
-
instance = await DuckDBInstance.create(dbPath === ':memory:' ? undefined : dbPath
|
|
117
|
+
instance = await DuckDBInstance.create(dbPath === ':memory:' ? undefined : dbPath, {
|
|
118
|
+
memory_limit: '512MB',
|
|
119
|
+
threads: '2',
|
|
120
|
+
});
|
|
118
121
|
con = await instance.connect();
|
|
119
122
|
readCon = await instance.connect();
|
|
120
123
|
for (const schema of tableSchemas) {
|
|
@@ -1325,41 +1328,3 @@ export async function filterTakendownDids(dids) {
|
|
|
1325
1328
|
const rows = await all(`SELECT did FROM _repos WHERE did IN (${placeholders}) AND status = 'takendown'`, ...dids);
|
|
1326
1329
|
return new Set(rows.map((r) => r.did));
|
|
1327
1330
|
}
|
|
1328
|
-
export async function backfillChildTables() {
|
|
1329
|
-
for (const [, schema] of schemas) {
|
|
1330
|
-
for (const child of schema.children) {
|
|
1331
|
-
// Check if child table needs backfill (significantly fewer rows than parent)
|
|
1332
|
-
const mainCount = (await all(`SELECT COUNT(*)::INTEGER as n FROM ${schema.tableName}`))[0]?.n || 0;
|
|
1333
|
-
if (mainCount === 0)
|
|
1334
|
-
continue;
|
|
1335
|
-
const childCount = (await all(`SELECT COUNT(DISTINCT parent_uri)::INTEGER as n FROM ${child.tableName}`))[0]?.n || 0;
|
|
1336
|
-
if (childCount >= mainCount * 0.9)
|
|
1337
|
-
continue;
|
|
1338
|
-
console.log(`[db] Backfilling ${child.tableName} from ${schema.tableName}...`);
|
|
1339
|
-
const snakeField = toSnakeCase(child.fieldName);
|
|
1340
|
-
const childColSelects = child.columns
|
|
1341
|
-
.map((c) => `json_extract_string(item.val, '$.${c.originalName}')`)
|
|
1342
|
-
.join(', ');
|
|
1343
|
-
const childColNames = ['parent_uri', 'parent_did', ...child.columns.map((c) => c.name)];
|
|
1344
|
-
const notNullFilters = child.columns
|
|
1345
|
-
.filter((c) => c.notNull)
|
|
1346
|
-
.map((c) => `json_extract_string(item.val, '$.${c.originalName}') IS NOT NULL`);
|
|
1347
|
-
const whereClause = [`p.${snakeField} IS NOT NULL`, ...notNullFilters].join(' AND ');
|
|
1348
|
-
try {
|
|
1349
|
-
await run(`DELETE FROM ${child.tableName}`);
|
|
1350
|
-
await run(`
|
|
1351
|
-
INSERT INTO ${child.tableName} (${childColNames.join(', ')})
|
|
1352
|
-
SELECT p.uri, p.did, ${childColSelects}
|
|
1353
|
-
FROM ${schema.tableName} p,
|
|
1354
|
-
unnest(from_json(p.${snakeField}::JSON, '["json"]')) AS item(val)
|
|
1355
|
-
WHERE ${whereClause}
|
|
1356
|
-
`);
|
|
1357
|
-
const result = await all(`SELECT COUNT(*)::INTEGER as n FROM ${child.tableName}`);
|
|
1358
|
-
console.log(`[db] Backfilled ${child.tableName}: ${result[0]?.n || 0} rows`);
|
|
1359
|
-
}
|
|
1360
|
-
catch (err) {
|
|
1361
|
-
console.warn(`[db] Backfill skipped for ${child.tableName}: ${err.message}`);
|
|
1362
|
-
}
|
|
1363
|
-
}
|
|
1364
|
-
}
|
|
1365
|
-
}
|
package/dist/main.js
CHANGED
|
@@ -5,7 +5,7 @@ import { log } from "./logger.js";
|
|
|
5
5
|
import { loadConfig } from "./config.js";
|
|
6
6
|
import { loadLexicons, storeLexicons, discoverCollections, generateTableSchema, generateCreateTableSQL, } from "./schema.js";
|
|
7
7
|
import { discoverViews } from "./views.js";
|
|
8
|
-
import { initDatabase, getCursor, querySQL
|
|
8
|
+
import { initDatabase, getCursor, querySQL } from "./db.js";
|
|
9
9
|
import { initFeeds, listFeeds } from "./feeds.js";
|
|
10
10
|
import { initXrpc, listXrpc, configureRelay } from "./xrpc.js";
|
|
11
11
|
import { initOpengraph } from "./opengraph.js";
|
|
@@ -75,8 +75,6 @@ if (config.database !== ':memory:') {
|
|
|
75
75
|
}
|
|
76
76
|
await initDatabase(config.database, schemas, ddlStatements);
|
|
77
77
|
log(`[main] DuckDB initialized (${config.database === ':memory:' ? 'in-memory' : config.database})`);
|
|
78
|
-
// 3a. Backfill child tables for decomposed arrays (one-time migration)
|
|
79
|
-
await backfillChildTables();
|
|
80
78
|
// 3b. Run setup hooks (after DB init, before server)
|
|
81
79
|
await initSetup(resolve(configDir, 'setup'));
|
|
82
80
|
// Detect orphaned tables
|