@hatk/hatk 0.0.1-alpha.11 → 0.0.1-alpha.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config.js +1 -1
- package/dist/db.d.ts +0 -1
- package/dist/db.d.ts.map +1 -1
- package/dist/db.js +4 -39
- package/dist/fts.d.ts.map +1 -1
- package/dist/fts.js +5 -0
- package/dist/main.js +9 -3
- package/package.json +1 -1
package/dist/config.js
CHANGED
|
@@ -27,7 +27,7 @@ export function loadConfig(configPath) {
|
|
|
27
27
|
fetchTimeout: parseInt(env.BACKFILL_FETCH_TIMEOUT || '') || backfillRaw.fetchTimeout || 300,
|
|
28
28
|
maxRetries: parseInt(env.BACKFILL_MAX_RETRIES || '') || backfillRaw.maxRetries || 5,
|
|
29
29
|
},
|
|
30
|
-
ftsRebuildInterval: parseInt(env.FTS_REBUILD_INTERVAL || '') || parsed.ftsRebuildInterval ||
|
|
30
|
+
ftsRebuildInterval: parseInt(env.FTS_REBUILD_INTERVAL || '') || parsed.ftsRebuildInterval || 5000,
|
|
31
31
|
oauth: null,
|
|
32
32
|
admins: env.ADMINS ? env.ADMINS.split(',').map((s) => s.trim()) : parsed.admins || [],
|
|
33
33
|
};
|
package/dist/db.d.ts
CHANGED
|
@@ -130,6 +130,5 @@ export declare function isTakendownDid(did: string): Promise<boolean>;
|
|
|
130
130
|
export declare function getPreferences(did: string): Promise<Record<string, any>>;
|
|
131
131
|
export declare function putPreference(did: string, key: string, value: any): Promise<void>;
|
|
132
132
|
export declare function filterTakendownDids(dids: string[]): Promise<Set<string>>;
|
|
133
|
-
export declare function backfillChildTables(): Promise<void>;
|
|
134
133
|
export {};
|
|
135
134
|
//# sourceMappingURL=db.d.ts.map
|
package/dist/db.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../src/db.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,WAAW,EAAe,MAAM,aAAa,CAAA;AAC3D,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,gBAAgB,CAAA;AAUzC,wBAAgB,aAAa,IAAI,IAAI,CAUpC;AA+DD,wBAAsB,QAAQ,CAAC,UAAU,EAAE,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,GAAG,EAAE,CAAA;CAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAkB/F;AAiBD,wBAAsB,YAAY,CAChC,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,WAAW,EAAE,EAC3B,aAAa,EAAE,MAAM,EAAE,GACtB,OAAO,CAAC,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../src/db.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,WAAW,EAAe,MAAM,aAAa,CAAA;AAC3D,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,gBAAgB,CAAA;AAUzC,wBAAgB,aAAa,IAAI,IAAI,CAUpC;AA+DD,wBAAsB,QAAQ,CAAC,UAAU,EAAE,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,GAAG,EAAE,CAAA;CAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAkB/F;AAiBD,wBAAsB,YAAY,CAChC,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,WAAW,EAAE,EAC3B,aAAa,EAAE,MAAM,EAAE,GACtB,OAAO,CAAC,IAAI,CAAC,CAoEf;AAED,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAGnE;AAED,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAEzE;AAED,wBAAsB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAGvE;AAED,wBAAsB,aAAa,CACjC,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,EACd,GAAG,CAAC,EAAE,MAAM,EACZ,IAAI,CAAC,EAAE;IAAE,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAC1E,OAAO,CAAC,IAAI,CAAC,CA0Cf;AAED,wBAAsB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAGpE;AAED,wBAAsB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,CAI9G;AAED,wBAAsB,sBAAsB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAQlF;AAED,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,CAG1D;AAED,wBAAsB,mBAAmB,IAAI,OAAO,CAAC,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC,CAE3F;AAED,wBAAsB,kBAAkB,CACtC,IAAI,GAAE;IACJ,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,CAAC,CAAC,EAAE,MAAM,CAAA;CACN,GACL,OAAO,CAAC;IAAE,KAAK,EAAE,GAAG,EAAE,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CA6B1C;AAED,wBAAsB,mBAAmB,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAO3E;AAED,wBAAsB,aAAa,IAAI,OAAO,CAAC,MAAM,CAAC,CAGrD;AAED,wBAAgB,aAAa,CAC3B,UAAU,EAAE,MAAM,EAClB,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAC1B;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,GAAG,EAAE,CAAA;CAAE,CA+BhC;AAED,wBAAsB,YAAY,CAChC,UAAU,EAAE,MAAM,EAClB,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAwGf;AAWD,wBAAsB,YAAY,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAYjF;AAED,wBAAsB,YAAY,CAChC,MAAM,EAAE,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,OAAO,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,GAClG,OAAO,CAAC,IAAI,CAAC,CAsBf;AAED,wBAAsB,kBAAkB,CACtC,IAAI,EAAE,MAAM,EAAE,GACb,OAAO,CACR,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,OAAO,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,CAAC,CAC7G,CAqBA;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,EAAE,MAAM,CAAA;IAClB,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;CAC5B;AAED,wBAAsB,iBAAiB,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAwQ9E;AAED,UAAU,SAAS;IACjB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAChC,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAA;CACvB;AAED,wBAAsB,YAAY,CAChC,UAAU,EAAE,MAAM,EAClB,IAAI,GAAE,SAAc,GACnB,OAAO,CAAC;IAAE,OAAO,EAAE,GAAG,EAAE,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,CAoF9C;AAED,wBAAsB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAgCrE;AAED,wBAAsB,gBAAgB,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAqCzF;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,aAAa,CACjC,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,EACb,IAAI,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,OAAO,CAAA;CAAO,GAC9D,OAAO,CAAC;IAAE,OAAO,EAAE,GAAG,EAAE,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,CAmN9C;AAGD,wBAAsB,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,GAAE,GAAG,EAAO,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAE9E;AAED,wBAAsB,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAEzE;AAED,wBAAgB,SAAS,CAAC,UAAU,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS,CAErE;AAED,wBAAsB,YAAY,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAKpG;AAED,wBAAsB,iBAAiB,CACrC,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EAAE,GACf,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAc9B;AAED,wBAAsB,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAKvG;AAED,wBAAsB,gBAAgB,CACpC,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EAAE,GACf,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CA6B7B;AAED,wBAAsB,kBAAkB,CACtC,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EAAE,GACf,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CASpC;AAED,wBAAsB,eAAe,CACnC,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,EAAE,GAC7C,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAOxB;AAKD,wBAAgB,cAAc,CAAC,CAAC,EAAE,GAAG,GAAG,GAAG,CAI1C;AAED,wBAAsB,YAAY,CAAC,cAAc,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CAW5G;AAED,wBAAgB,UAAU,CACxB,GAAG,EAAE,GAAG,EACR,SAAS,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,EAC3C,SAAS,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,GACvD,GAAG,CAAC,OAAO,CAAC,GAAG,IAAI,CAiGrB;AAED,wBAAgB,UAAU,CAAC,OAAO,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAGhE;AAED,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CASpF;AAED,wBAAsB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAKlE;AAED,wBAAsB,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,GAAE,MAAW,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAMtF;AAED,wBAAsB,qBAAqB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAOxE;AAED,wBAAsB,sBAAsB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAO3E;AAED,wBAAsB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAGlE;AAED,wBAAsB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAW9E;AAED,wBAAsB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAQvF;AAED,wBAAsB,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAK9E"}
|
package/dist/db.js
CHANGED
|
@@ -114,7 +114,10 @@ async function all(sql, ...params) {
|
|
|
114
114
|
return enqueue('read', () => allDirect(sql, ...params));
|
|
115
115
|
}
|
|
116
116
|
export async function initDatabase(dbPath, tableSchemas, ddlStatements) {
|
|
117
|
-
instance = await DuckDBInstance.create(dbPath === ':memory:' ? undefined : dbPath
|
|
117
|
+
instance = await DuckDBInstance.create(dbPath === ':memory:' ? undefined : dbPath, {
|
|
118
|
+
memory_limit: '256MB',
|
|
119
|
+
threads: '1',
|
|
120
|
+
});
|
|
118
121
|
con = await instance.connect();
|
|
119
122
|
readCon = await instance.connect();
|
|
120
123
|
for (const schema of tableSchemas) {
|
|
@@ -1325,41 +1328,3 @@ export async function filterTakendownDids(dids) {
|
|
|
1325
1328
|
const rows = await all(`SELECT did FROM _repos WHERE did IN (${placeholders}) AND status = 'takendown'`, ...dids);
|
|
1326
1329
|
return new Set(rows.map((r) => r.did));
|
|
1327
1330
|
}
|
|
1328
|
-
export async function backfillChildTables() {
|
|
1329
|
-
for (const [, schema] of schemas) {
|
|
1330
|
-
for (const child of schema.children) {
|
|
1331
|
-
// Check if child table needs backfill (significantly fewer rows than parent)
|
|
1332
|
-
const mainCount = (await all(`SELECT COUNT(*)::INTEGER as n FROM ${schema.tableName}`))[0]?.n || 0;
|
|
1333
|
-
if (mainCount === 0)
|
|
1334
|
-
continue;
|
|
1335
|
-
const childCount = (await all(`SELECT COUNT(DISTINCT parent_uri)::INTEGER as n FROM ${child.tableName}`))[0]?.n || 0;
|
|
1336
|
-
if (childCount >= mainCount * 0.9)
|
|
1337
|
-
continue;
|
|
1338
|
-
console.log(`[db] Backfilling ${child.tableName} from ${schema.tableName}...`);
|
|
1339
|
-
const snakeField = toSnakeCase(child.fieldName);
|
|
1340
|
-
const childColSelects = child.columns
|
|
1341
|
-
.map((c) => `json_extract_string(item.val, '$.${c.originalName}')`)
|
|
1342
|
-
.join(', ');
|
|
1343
|
-
const childColNames = ['parent_uri', 'parent_did', ...child.columns.map((c) => c.name)];
|
|
1344
|
-
const notNullFilters = child.columns
|
|
1345
|
-
.filter((c) => c.notNull)
|
|
1346
|
-
.map((c) => `json_extract_string(item.val, '$.${c.originalName}') IS NOT NULL`);
|
|
1347
|
-
const whereClause = [`p.${snakeField} IS NOT NULL`, ...notNullFilters].join(' AND ');
|
|
1348
|
-
try {
|
|
1349
|
-
await run(`DELETE FROM ${child.tableName}`);
|
|
1350
|
-
await run(`
|
|
1351
|
-
INSERT INTO ${child.tableName} (${childColNames.join(', ')})
|
|
1352
|
-
SELECT p.uri, p.did, ${childColSelects}
|
|
1353
|
-
FROM ${schema.tableName} p,
|
|
1354
|
-
unnest(from_json(p.${snakeField}::JSON, '["json"]')) AS item(val)
|
|
1355
|
-
WHERE ${whereClause}
|
|
1356
|
-
`);
|
|
1357
|
-
const result = await all(`SELECT COUNT(*)::INTEGER as n FROM ${child.tableName}`);
|
|
1358
|
-
console.log(`[db] Backfilled ${child.tableName}: ${result[0]?.n || 0} rows`);
|
|
1359
|
-
}
|
|
1360
|
-
catch (err) {
|
|
1361
|
-
console.warn(`[db] Backfill skipped for ${child.tableName}: ${err.message}`);
|
|
1362
|
-
}
|
|
1363
|
-
}
|
|
1364
|
-
}
|
|
1365
|
-
}
|
package/dist/fts.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fts.d.ts","sourceRoot":"","sources":["../src/fts.ts"],"names":[],"mappings":"AAwEA,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CAE7D;AAED,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAElE;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAEvD;AAED;;;;GAIG;AACH,wBAAsB,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CA0FrE;AAokBD;;;GAGG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAIpD;AAED,wBAAsB,iBAAiB,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"fts.d.ts","sourceRoot":"","sources":["../src/fts.ts"],"names":[],"mappings":"AAwEA,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CAE7D;AAED,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAElE;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAEvD;AAED;;;;GAIG;AACH,wBAAsB,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CA0FrE;AAokBD;;;GAGG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAIpD;AAED,wBAAsB,iBAAiB,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CA0B5E"}
|
package/dist/fts.js
CHANGED
|
@@ -752,6 +752,11 @@ export async function rebuildAllIndexes(collections) {
|
|
|
752
752
|
errors.push(`${collection}: ${err.message}`);
|
|
753
753
|
}
|
|
754
754
|
}
|
|
755
|
+
// Compact WAL to free DuckDB memory after heavy FTS operations
|
|
756
|
+
try {
|
|
757
|
+
await runSQL('CHECKPOINT');
|
|
758
|
+
}
|
|
759
|
+
catch { }
|
|
755
760
|
emit('fts', 'rebuild', {
|
|
756
761
|
collections_total: collections.length,
|
|
757
762
|
collections_rebuilt: rebuilt,
|
package/dist/main.js
CHANGED
|
@@ -5,7 +5,7 @@ import { log } from "./logger.js";
|
|
|
5
5
|
import { loadConfig } from "./config.js";
|
|
6
6
|
import { loadLexicons, storeLexicons, discoverCollections, generateTableSchema, generateCreateTableSQL, } from "./schema.js";
|
|
7
7
|
import { discoverViews } from "./views.js";
|
|
8
|
-
import { initDatabase, getCursor, querySQL
|
|
8
|
+
import { initDatabase, getCursor, querySQL } from "./db.js";
|
|
9
9
|
import { initFeeds, listFeeds } from "./feeds.js";
|
|
10
10
|
import { initXrpc, listXrpc, configureRelay } from "./xrpc.js";
|
|
11
11
|
import { initOpengraph } from "./opengraph.js";
|
|
@@ -19,8 +19,13 @@ import { runBackfill } from "./backfill.js";
|
|
|
19
19
|
import { initOAuth } from "./oauth/server.js";
|
|
20
20
|
import { loadOnLoginHook } from "./oauth/hooks.js";
|
|
21
21
|
import { initSetup } from "./setup.js";
|
|
22
|
+
function logMemory(phase) {
|
|
23
|
+
const mem = process.memoryUsage();
|
|
24
|
+
log(`[mem] ${phase}: heap=${Math.round(mem.heapUsed / 1024 / 1024)}MB rss=${Math.round(mem.rss / 1024 / 1024)}MB external=${Math.round(mem.external / 1024 / 1024)}MB arrayBuffers=${Math.round(mem.arrayBuffers / 1024 / 1024)}MB`);
|
|
25
|
+
}
|
|
22
26
|
const configPath = process.argv[2] || 'config.yaml';
|
|
23
27
|
const configDir = dirname(resolve(configPath));
|
|
28
|
+
logMemory('startup');
|
|
24
29
|
// 1. Load config
|
|
25
30
|
const config = loadConfig(configPath);
|
|
26
31
|
configureRelay(config.relay);
|
|
@@ -74,9 +79,8 @@ if (config.database !== ':memory:') {
|
|
|
74
79
|
mkdirSync(dirname(config.database), { recursive: true });
|
|
75
80
|
}
|
|
76
81
|
await initDatabase(config.database, schemas, ddlStatements);
|
|
82
|
+
logMemory('after-db-init');
|
|
77
83
|
log(`[main] DuckDB initialized (${config.database === ':memory:' ? 'in-memory' : config.database})`);
|
|
78
|
-
// 3a. Backfill child tables for decomposed arrays (one-time migration)
|
|
79
|
-
await backfillChildTables();
|
|
80
84
|
// 3b. Run setup hooks (after DB init, before server)
|
|
81
85
|
await initSetup(resolve(configDir, 'setup'));
|
|
82
86
|
// Detect orphaned tables
|
|
@@ -106,6 +110,7 @@ if (config.oauth) {
|
|
|
106
110
|
await initOAuth(config.oauth, config.plc, config.relay);
|
|
107
111
|
log(`[main] OAuth initialized (issuer: ${config.oauth.issuer})`);
|
|
108
112
|
}
|
|
113
|
+
logMemory('before-server');
|
|
109
114
|
// 5. Start server immediately (don't wait for backfill)
|
|
110
115
|
const collectionSet = new Set(collections);
|
|
111
116
|
startServer(config.port, collections, config.publicDir, config.oauth, config.admins);
|
|
@@ -117,6 +122,7 @@ log(` Collections: ${collections.join(', ')}`);
|
|
|
117
122
|
log(` Feeds: ${listFeeds()
|
|
118
123
|
.map((f) => f.name)
|
|
119
124
|
.join(', ')}`);
|
|
125
|
+
logMemory('after-server');
|
|
120
126
|
// 6. Start indexer with cursor
|
|
121
127
|
const cursor = await getCursor('relay');
|
|
122
128
|
startIndexer({
|