@hatk/hatk 0.0.1-alpha.3 → 0.0.1-alpha.31

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/dist/adapter.d.ts +19 -0
  2. package/dist/adapter.d.ts.map +1 -0
  3. package/dist/adapter.js +94 -0
  4. package/dist/backfill.d.ts +60 -1
  5. package/dist/backfill.d.ts.map +1 -1
  6. package/dist/backfill.js +166 -32
  7. package/dist/car.d.ts +59 -1
  8. package/dist/car.d.ts.map +1 -1
  9. package/dist/car.js +179 -7
  10. package/dist/cbor.d.ts +37 -0
  11. package/dist/cbor.d.ts.map +1 -1
  12. package/dist/cbor.js +36 -3
  13. package/dist/cid.d.ts +37 -0
  14. package/dist/cid.d.ts.map +1 -1
  15. package/dist/cid.js +38 -3
  16. package/dist/cli.js +399 -130
  17. package/dist/config.d.ts +12 -1
  18. package/dist/config.d.ts.map +1 -1
  19. package/dist/config.js +36 -9
  20. package/dist/database/adapter-factory.d.ts +6 -0
  21. package/dist/database/adapter-factory.d.ts.map +1 -0
  22. package/dist/database/adapter-factory.js +20 -0
  23. package/dist/database/adapters/duckdb-search.d.ts +12 -0
  24. package/dist/database/adapters/duckdb-search.d.ts.map +1 -0
  25. package/dist/database/adapters/duckdb-search.js +27 -0
  26. package/dist/database/adapters/duckdb.d.ts +25 -0
  27. package/dist/database/adapters/duckdb.d.ts.map +1 -0
  28. package/dist/database/adapters/duckdb.js +161 -0
  29. package/dist/database/adapters/sqlite-search.d.ts +23 -0
  30. package/dist/database/adapters/sqlite-search.d.ts.map +1 -0
  31. package/dist/database/adapters/sqlite-search.js +71 -0
  32. package/dist/database/adapters/sqlite.d.ts +18 -0
  33. package/dist/database/adapters/sqlite.d.ts.map +1 -0
  34. package/dist/database/adapters/sqlite.js +87 -0
  35. package/dist/database/db.d.ts +149 -0
  36. package/dist/database/db.d.ts.map +1 -0
  37. package/dist/database/db.js +1464 -0
  38. package/dist/database/dialect.d.ts +45 -0
  39. package/dist/database/dialect.d.ts.map +1 -0
  40. package/dist/database/dialect.js +72 -0
  41. package/dist/database/fts.d.ts +27 -0
  42. package/dist/database/fts.d.ts.map +1 -0
  43. package/dist/database/fts.js +846 -0
  44. package/dist/database/index.d.ts +7 -0
  45. package/dist/database/index.d.ts.map +1 -0
  46. package/dist/database/index.js +6 -0
  47. package/dist/database/ports.d.ts +50 -0
  48. package/dist/database/ports.d.ts.map +1 -0
  49. package/dist/database/ports.js +1 -0
  50. package/dist/database/schema.d.ts +60 -0
  51. package/dist/database/schema.d.ts.map +1 -0
  52. package/dist/database/schema.js +388 -0
  53. package/dist/db.d.ts +1 -1
  54. package/dist/db.d.ts.map +1 -1
  55. package/dist/db.js +4 -38
  56. package/dist/dev-entry.d.ts +8 -0
  57. package/dist/dev-entry.d.ts.map +1 -0
  58. package/dist/dev-entry.js +109 -0
  59. package/dist/feeds.d.ts +4 -0
  60. package/dist/feeds.d.ts.map +1 -1
  61. package/dist/feeds.js +42 -3
  62. package/dist/fts.d.ts.map +1 -1
  63. package/dist/fts.js +5 -0
  64. package/dist/hooks.d.ts +22 -0
  65. package/dist/hooks.d.ts.map +1 -0
  66. package/dist/hooks.js +75 -0
  67. package/dist/hydrate.js +1 -1
  68. package/dist/indexer.d.ts +20 -0
  69. package/dist/indexer.d.ts.map +1 -1
  70. package/dist/indexer.js +53 -7
  71. package/dist/labels.d.ts +34 -0
  72. package/dist/labels.d.ts.map +1 -1
  73. package/dist/labels.js +63 -3
  74. package/dist/logger.d.ts +29 -0
  75. package/dist/logger.d.ts.map +1 -1
  76. package/dist/logger.js +29 -0
  77. package/dist/main.js +138 -67
  78. package/dist/mst.d.ts +18 -1
  79. package/dist/mst.d.ts.map +1 -1
  80. package/dist/mst.js +19 -8
  81. package/dist/oauth/db.d.ts.map +1 -1
  82. package/dist/oauth/db.js +41 -15
  83. package/dist/oauth/server.d.ts +2 -0
  84. package/dist/oauth/server.d.ts.map +1 -1
  85. package/dist/oauth/server.js +102 -7
  86. package/dist/oauth/session.d.ts +9 -0
  87. package/dist/oauth/session.d.ts.map +1 -0
  88. package/dist/oauth/session.js +65 -0
  89. package/dist/opengraph.d.ts +10 -0
  90. package/dist/opengraph.d.ts.map +1 -1
  91. package/dist/opengraph.js +103 -5
  92. package/dist/pds-proxy.d.ts +39 -0
  93. package/dist/pds-proxy.d.ts.map +1 -0
  94. package/dist/pds-proxy.js +173 -0
  95. package/dist/renderer.d.ts +27 -0
  96. package/dist/renderer.d.ts.map +1 -0
  97. package/dist/renderer.js +46 -0
  98. package/dist/resolve-hatk.d.ts +6 -0
  99. package/dist/resolve-hatk.d.ts.map +1 -0
  100. package/dist/resolve-hatk.js +20 -0
  101. package/dist/response.d.ts +16 -0
  102. package/dist/response.d.ts.map +1 -0
  103. package/dist/response.js +69 -0
  104. package/dist/scanner.d.ts +21 -0
  105. package/dist/scanner.d.ts.map +1 -0
  106. package/dist/scanner.js +88 -0
  107. package/dist/schema.d.ts +8 -0
  108. package/dist/schema.d.ts.map +1 -1
  109. package/dist/schema.js +29 -0
  110. package/dist/seed.d.ts +19 -0
  111. package/dist/seed.d.ts.map +1 -1
  112. package/dist/seed.js +43 -4
  113. package/dist/server-init.d.ts +8 -0
  114. package/dist/server-init.d.ts.map +1 -0
  115. package/dist/server-init.js +59 -0
  116. package/dist/server.d.ts +26 -3
  117. package/dist/server.d.ts.map +1 -1
  118. package/dist/server.js +487 -616
  119. package/dist/setup.d.ts +28 -1
  120. package/dist/setup.d.ts.map +1 -1
  121. package/dist/setup.js +50 -3
  122. package/dist/test.d.ts +1 -1
  123. package/dist/test.d.ts.map +1 -1
  124. package/dist/test.js +38 -32
  125. package/dist/views.js +1 -1
  126. package/dist/vite-plugin.d.ts +1 -1
  127. package/dist/vite-plugin.d.ts.map +1 -1
  128. package/dist/vite-plugin.js +252 -66
  129. package/dist/xrpc.d.ts +36 -0
  130. package/dist/xrpc.d.ts.map +1 -1
  131. package/dist/xrpc.js +124 -3
  132. package/package.json +13 -6
@@ -0,0 +1,109 @@
1
+ /**
2
+ * Dev mode entry point — loaded through Vite's module runner.
3
+ * Boots hatk infrastructure and exports the fetch handler.
4
+ */
5
+ import { loadConfig } from "./config.js";
6
+ import { loadLexicons, storeLexicons, discoverCollections, buildSchemas } from "./database/schema.js";
7
+ import { discoverViews } from "./views.js";
8
+ import { initDatabase, migrateSchema, getSchemaDump } from "./database/db.js";
9
+ import { createAdapter } from "./database/adapter-factory.js";
10
+ import { getDialect } from "./database/dialect.js";
11
+ import { setSearchPort } from "./database/fts.js";
12
+ import { configureRelay } from "./xrpc.js";
13
+ import { initOAuth } from "./oauth/server.js";
14
+ import { initServer } from "./server-init.js";
15
+ import { createHandler, registerCoreHandlers } from "./server.js";
16
+ import { startIndexer } from "./indexer.js";
17
+ import { getCursor } from "./database/db.js";
18
+ import { runBackfill } from "./backfill.js";
19
+ import { rebuildAllIndexes } from "./database/fts.js";
20
+ import { relayHttpUrl } from "./config.js";
21
+ import { validateLexicons } from '@bigmoves/lexicon';
22
+ import { log } from "./logger.js";
23
+ import { mkdirSync } from 'node:fs';
24
+ import { dirname, resolve } from 'node:path';
25
+ process.env.DEV_MODE = '1';
26
+ // Boot sequence (mirrors main.ts but exports handler instead of starting server)
27
+ const configPath = 'hatk.config.ts';
28
+ const configDir = dirname(resolve(configPath));
29
+ const config = await loadConfig(configPath);
30
+ configureRelay(config.relay);
31
+ const lexicons = loadLexicons(resolve(configDir, 'lexicons'));
32
+ const lexiconErrors = validateLexicons([...lexicons.values()]);
33
+ if (lexiconErrors) {
34
+ for (const [nsid, errors] of Object.entries(lexiconErrors)) {
35
+ for (const err of errors)
36
+ console.error(`Invalid lexicon ${nsid}: ${err}`);
37
+ }
38
+ throw new Error('Invalid lexicons');
39
+ }
40
+ storeLexicons(lexicons);
41
+ const collections = config.collections.length > 0 ? config.collections : discoverCollections(lexicons);
42
+ discoverViews();
43
+ const engineDialect = getDialect(config.databaseEngine);
44
+ const { schemas, ddlStatements } = buildSchemas(lexicons, collections, engineDialect);
45
+ if (config.database !== ':memory:') {
46
+ mkdirSync(dirname(config.database), { recursive: true });
47
+ }
48
+ const { adapter, searchPort } = await createAdapter(config.databaseEngine);
49
+ setSearchPort(searchPort);
50
+ await initDatabase(adapter, config.database, schemas, ddlStatements);
51
+ await migrateSchema(schemas);
52
+ // Write db/schema.sql
53
+ try {
54
+ const { mkdirSync, writeFileSync } = await import('node:fs');
55
+ const schemaDir = resolve(configDir, 'db');
56
+ mkdirSync(schemaDir, { recursive: true });
57
+ const schemaDump = await getSchemaDump();
58
+ writeFileSync(resolve(schemaDir, 'schema.sql'), `-- This file is auto-generated by hatk on startup. Do not edit.\n-- Database engine: ${config.databaseEngine}\n\n${schemaDump}\n`);
59
+ log(`[hatk] Schema written to db/schema.sql`);
60
+ }
61
+ catch { }
62
+ // Initialize handlers from server/ directory
63
+ await initServer(resolve(configDir, 'server'));
64
+ // Register built-in dev.hatk.* handlers so callXrpc() can find them
65
+ registerCoreHandlers(collections, config.oauth);
66
+ if (config.oauth) {
67
+ await initOAuth(config.oauth, config.plc, config.relay);
68
+ }
69
+ // Start indexer
70
+ const collectionSet = new Set(collections);
71
+ const cursor = await getCursor('relay');
72
+ startIndexer({
73
+ relayUrl: config.relay,
74
+ collections: collectionSet,
75
+ signalCollections: config.backfill.signalCollections ? new Set(config.backfill.signalCollections) : undefined,
76
+ pinnedRepos: config.backfill.repos ? new Set(config.backfill.repos) : undefined,
77
+ cursor,
78
+ fetchTimeout: config.backfill.fetchTimeout,
79
+ maxRetries: config.backfill.maxRetries,
80
+ parallelism: config.backfill.parallelism,
81
+ ftsRebuildInterval: config.ftsRebuildInterval,
82
+ });
83
+ // Run backfill in background (no restart in dev mode)
84
+ runBackfill({
85
+ pdsUrl: relayHttpUrl(config.relay),
86
+ plcUrl: config.plc,
87
+ collections: collectionSet,
88
+ config: config.backfill,
89
+ }).then(() => rebuildAllIndexes(Array.from(collectionSet)))
90
+ .catch((err) => console.error('[backfill]', err.message));
91
+ // Export the handler for Vite middleware
92
+ export const handler = createHandler({
93
+ collections: Array.from(collectionSet),
94
+ publicDir: null, // Vite serves static assets in dev
95
+ oauth: config.oauth,
96
+ admins: config.admins,
97
+ });
98
+ /** Re-scan server/ directory to pick up changed handlers in dev mode. */
99
+ export async function reloadServer() {
100
+ await initServer(resolve(configDir, 'server'));
101
+ }
102
+ export { renderPage } from "./renderer.js";
103
+ export { getRenderer } from "./renderer.js";
104
+ export { callXrpc } from "./xrpc.js";
105
+ export { parseSessionCookie, getSessionCookieName } from "./oauth/session.js";
106
+ log(`[hatk] Dev server ready`);
107
+ log(` Relay: ${config.relay}`);
108
+ log(` Database: ${config.database}`);
109
+ log(` Collections: ${collections.join(', ')}`);
package/dist/feeds.d.ts CHANGED
@@ -71,13 +71,17 @@ export declare function defineFeed(opts: FeedOpts): {
71
71
  view?: string;
72
72
  label: string;
73
73
  hydrate?: (ctx: HydrateContext<any>) => Promise<unknown[]>;
74
+ __type: "feed";
74
75
  } | {
75
76
  generate: (ctx: any) => Promise<Checked<FeedResult>>;
76
77
  collection?: never;
77
78
  view?: never;
78
79
  label: string;
79
80
  hydrate: (ctx: HydrateContext<any>) => Promise<unknown[]>;
81
+ __type: "feed";
80
82
  };
83
+ /** Register a single feed from a scanned server/ module. */
84
+ export declare function registerFeed(name: string, generator: ReturnType<typeof defineFeed>): void;
81
85
  export declare function initFeeds(feedsDir: string): Promise<void>;
82
86
  /** Execute a feed and run its hydrate pipeline if present. */
83
87
  export declare function executeFeed(name: string, params: Record<string, string>, cursor: string | undefined, limit: number, viewer?: {
@@ -1 +1 @@
1
- {"version":3,"file":"feeds.d.ts","sourceRoot":"","sources":["../src/feeds.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,cAAc,EAAE,GAAG,EAAE,MAAM,cAAc,CAAA;AACvD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAA;AAE7C,YAAY,EAAE,cAAc,EAAE,GAAG,EAAE,CAAA;AAEnC,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,EAAE,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,EAAE,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,CAAA;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAA;CACvB;AAED,MAAM,WAAW,cAAc,CAAC,CAAC;IAC/B,IAAI,EAAE,CAAC,EAAE,CAAA;IACT,MAAM,EAAE,MAAM,GAAG,SAAS,CAAA;CAC3B;AAED,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE;QAAE,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,GAAG,EAAE,KAAK,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;KAAE,CAAA;IAC9D,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAA;IAC9B,UAAU,EAAE,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,MAAM,CAAA;IAC7D,YAAY,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAA;IACzE,WAAW,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAA;IAC9C,mBAAmB,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAA;IAC7D,QAAQ,EAAE,CAAC,CAAC,SAAS;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,YAAY,KAAK,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAA;CACtG;AAkBD,KAAK,YAAY,GAAG,CAClB,GAAG,EAAE,WAAW,GAAG;IAAE,EAAE,EAAE,CAAC,KAAK,EAAE,UAAU,KAAK,OAAO,CAAC,UAAU,CAAC,CAAA;CAAE,KAClE,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAA;AAEjC,KAAK,QAAQ,GACT;IACE,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,YAAY,CAAA;IACtB,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,cAAc,CAAC,GAAG,CAAC,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;CAC3D,GACD;IACE,UAAU,CAAC,EAAE,KAAK,CAAA;IAClB,IAAI,CAAC,EAAE,KAAK,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,YAAY,CAAA;IACtB,OAAO,EAAE,CAAC,GAAG,EAAE,cAAc,CAAC,GAAG,CAAC,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;CAC1D,CAAA;AAEL,wBAAgB,cAAc,CAAC,IAAI,EAAE;IACnC,EAAE,EAAE;QAAE,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,GAAG,EAAE,KAAK,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;KAAE,CAAA;IAC9D,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,UAAU,EAAE,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,MAAM,CAAA;IAC7D,YAAY,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAA;CAC1E,IACe,CAAC,SAAS;IAAE,GAAG,EAAE,MAAM,CAAA;CAAE,EAAE,KAAK,MAAM,EAAE,OAAO,YAAY,KAAG,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAmDvG;AAED,wBAAgB,UAAU,CAAC,IAAI,EAAE,QAAQ;oBACL,GAAG;gBA3ErB,MAAM;WACX,MAAM;WACN,MAAM;cAEH,CAAC,GAAG,EAAE,cAAc,CAAC,GAAG,CAAC,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC;;oBAuE5B,GAAG;iBApEpB,KAAK;WACX,KAAK;WACL,MAAM;aAEJ,CAAC,GAAG,EAAE,cAAc,CAAC,GAAG,CAAC,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC;EAiE9D;AAID,wBAAsB,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CA8D/D;AAED,8DAA8D;AAC9D,wBAAsB,WAAW,CAC/B,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAC9B,MAAM,EAAE,MAAM,GAAG,SAAS,EAC1B,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE;IAAE,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,GAC9B,OAAO,CAAC;IAAE,KAAK,CAAC,EAAE,OAAO,EAAE,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,CAczE;AAED,wBAAgB,SAAS,IAAI;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,EAAE,CAE7D;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAEpD"}
1
+ {"version":3,"file":"feeds.d.ts","sourceRoot":"","sources":["../src/feeds.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,cAAc,EAAE,GAAG,EAAE,MAAM,cAAc,CAAA;AACvD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAA;AAE7C,YAAY,EAAE,cAAc,EAAE,GAAG,EAAE,CAAA;AAEnC,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,EAAE,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,EAAE,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,CAAA;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAA;CACvB;AAED,MAAM,WAAW,cAAc,CAAC,CAAC;IAC/B,IAAI,EAAE,CAAC,EAAE,CAAA;IACT,MAAM,EAAE,MAAM,GAAG,SAAS,CAAA;CAC3B;AAED,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE;QAAE,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,GAAG,EAAE,KAAK,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;KAAE,CAAA;IAC9D,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAA;IAC9B,UAAU,EAAE,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,MAAM,CAAA;IAC7D,YAAY,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAA;IACzE,WAAW,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAA;IAC9C,mBAAmB,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAA;IAC7D,QAAQ,EAAE,CAAC,CAAC,SAAS;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,YAAY,KAAK,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAA;CACtG;AAkBD,KAAK,YAAY,GAAG,CAClB,GAAG,EAAE,WAAW,GAAG;IAAE,EAAE,EAAE,CAAC,KAAK,EAAE,UAAU,KAAK,OAAO,CAAC,UAAU,CAAC,CAAA;CAAE,KAClE,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAA;AAEjC,KAAK,QAAQ,GACT;IACE,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,YAAY,CAAA;IACtB,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,cAAc,CAAC,GAAG,CAAC,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;CAC3D,GACD;IACE,UAAU,CAAC,EAAE,KAAK,CAAA;IAClB,IAAI,CAAC,EAAE,KAAK,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,YAAY,CAAA;IACtB,OAAO,EAAE,CAAC,GAAG,EAAE,cAAc,CAAC,GAAG,CAAC,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;CAC1D,CAAA;AAEL,wBAAgB,cAAc,CAAC,IAAI,EAAE;IACnC,EAAE,EAAE;QAAE,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,GAAG,EAAE,KAAK,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;KAAE,CAAA;IAC9D,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,UAAU,EAAE,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,MAAM,CAAA;IAC7D,YAAY,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAA;CAC1E,IACe,CAAC,SAAS;IAAE,GAAG,EAAE,MAAM,CAAA;CAAE,EAAE,KAAK,MAAM,EAAE,OAAO,YAAY,KAAG,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAmDvG;AAED,wBAAgB,UAAU,CAAC,IAAI,EAAE,QAAQ;oBACoB,GAAG;gBA3E9C,MAAM;WACX,MAAM;WACN,MAAM;cAEH,CAAC,GAAG,EAAE,cAAc,CAAC,GAAG,CAAC,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC;;;oBAuEH,GAAG;iBApE7C,KAAK;WACX,KAAK;WACL,MAAM;aAEJ,CAAC,GAAG,EAAE,cAAc,CAAC,GAAG,CAAC,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC;;EAiE9D;AAED,4DAA4D;AAC5D,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,CAAC,OAAO,UAAU,CAAC,GAAG,IAAI,CAuCzF;AAID,wBAAsB,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CA8D/D;AAED,8DAA8D;AAC9D,wBAAsB,WAAW,CAC/B,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAC9B,MAAM,EAAE,MAAM,GAAG,SAAS,EAC1B,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE;IAAE,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,GAC9B,OAAO,CAAC;IAAE,KAAK,CAAC,EAAE,OAAO,EAAE,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,CAczE;AAED,wBAAgB,SAAS,IAAI;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,EAAE,CAE7D;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAEpD"}
package/dist/feeds.js CHANGED
@@ -9,7 +9,7 @@ var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExte
9
9
  import { resolve } from 'node:path';
10
10
  import { readdirSync } from 'node:fs';
11
11
  import { log } from "./logger.js";
12
- import { querySQL, packCursor, unpackCursor, isTakendownDid, filterTakendownDids } from "./db.js";
12
+ import { querySQL, packCursor, unpackCursor, isTakendownDid, filterTakendownDids } from "./database/db.js";
13
13
  import { resolveRecords, buildHydrateContext } from "./hydrate.js";
14
14
  export function createPaginate(deps) {
15
15
  return async (sql, opts) => {
@@ -59,7 +59,46 @@ export function createPaginate(deps) {
59
59
  };
60
60
  }
61
61
  export function defineFeed(opts) {
62
- return { ...opts, generate: (ctx) => opts.generate({ ...ctx, ok: (v) => v }) };
62
+ return { __type: 'feed', ...opts, generate: (ctx) => opts.generate({ ...ctx, ok: (v) => v }) };
63
+ }
64
+ /** Register a single feed from a scanned server/ module. */
65
+ export function registerFeed(name, generator) {
66
+ const handler = {
67
+ name,
68
+ label: generator.label || name,
69
+ collection: generator.collection,
70
+ view: generator.view,
71
+ generate: async (params, cursor, limit, viewer) => {
72
+ const paginateDeps = {
73
+ db: { query: querySQL },
74
+ cursor,
75
+ limit,
76
+ packCursor,
77
+ unpackCursor,
78
+ };
79
+ const ctx = {
80
+ db: { query: querySQL },
81
+ params,
82
+ cursor,
83
+ limit,
84
+ viewer,
85
+ packCursor,
86
+ unpackCursor,
87
+ isTakendown: isTakendownDid,
88
+ filterTakendownDids,
89
+ paginate: createPaginate(paginateDeps),
90
+ };
91
+ const result = await generator.generate(ctx);
92
+ if (Array.isArray(result)) {
93
+ return { uris: result.map((r) => r.uri || r) };
94
+ }
95
+ return { uris: result.uris, cursor: result.cursor };
96
+ },
97
+ };
98
+ if (typeof generator.hydrate === 'function') {
99
+ handler.hydrate = generator.hydrate;
100
+ }
101
+ feeds.set(name, handler);
63
102
  }
64
103
  const feeds = new Map();
65
104
  export async function initFeeds(feedsDir) {
@@ -75,7 +114,7 @@ export async function initFeeds(feedsDir) {
75
114
  for (const file of files) {
76
115
  const name = file.replace(/\.(ts|js)$/, '');
77
116
  const scriptPath = resolve(feedsDir, file);
78
- const mod = await import(__rewriteRelativeImportExtension(scriptPath));
117
+ const mod = await import(__rewriteRelativeImportExtension(/* @vite-ignore */ `${scriptPath}?t=${Date.now()}`));
79
118
  const generator = mod.default;
80
119
  const handler = {
81
120
  name,
package/dist/fts.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"fts.d.ts","sourceRoot":"","sources":["../src/fts.ts"],"names":[],"mappings":"AAwEA,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CAE7D;AAED,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAElE;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAEvD;AAED;;;;GAIG;AACH,wBAAsB,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CA0FrE;AAokBD;;;GAGG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAIpD;AAED,wBAAsB,iBAAiB,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAqB5E"}
1
+ {"version":3,"file":"fts.d.ts","sourceRoot":"","sources":["../src/fts.ts"],"names":[],"mappings":"AAwEA,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CAE7D;AAED,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAElE;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAEvD;AAED;;;;GAIG;AACH,wBAAsB,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CA0FrE;AAokBD;;;GAGG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAIpD;AAED,wBAAsB,iBAAiB,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CA0B5E"}
package/dist/fts.js CHANGED
@@ -752,6 +752,11 @@ export async function rebuildAllIndexes(collections) {
752
752
  errors.push(`${collection}: ${err.message}`);
753
753
  }
754
754
  }
755
+ // Compact WAL to free DuckDB memory after heavy FTS operations
756
+ try {
757
+ await runSQL('CHECKPOINT');
758
+ }
759
+ catch { }
755
760
  emit('fts', 'rebuild', {
756
761
  collections_total: collections.length,
757
762
  collections_rebuilt: rebuilt,
@@ -0,0 +1,22 @@
1
+ /** Context passed to the on-login hook after a successful OAuth login. */
2
+ export type OnLoginCtx = {
3
+ /** DID of the user who just logged in. */
4
+ did: string;
5
+ /** Trigger a backfill for a DID if it hasn't been indexed yet. */
6
+ ensureRepo: (did: string) => Promise<void>;
7
+ };
8
+ export declare function defineHook(event: 'on-login', handler: (ctx: OnLoginCtx) => Promise<void>): {
9
+ __type: "hook";
10
+ event: "on-login";
11
+ handler: (ctx: OnLoginCtx) => Promise<void>;
12
+ };
13
+ /**
14
+ * Discover and load the on-login hook from the project's `hooks/` directory.
15
+ * Looks for `on-login.ts` or `on-login.js`. Safe to call if no hook exists.
16
+ */
17
+ export declare function loadOnLoginHook(hooksDir: string): Promise<void>;
18
+ /** Register a hook from a scanned server/ module. */
19
+ export declare function registerHook(event: string, handler: Function): void;
20
+ /** Fire the on-login hook if loaded. Errors are logged but never block login. */
21
+ export declare function fireOnLoginHook(did: string): Promise<void>;
22
+ //# sourceMappingURL=hooks.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"hooks.d.ts","sourceRoot":"","sources":["../src/hooks.ts"],"names":[],"mappings":"AA2BA,0EAA0E;AAC1E,MAAM,MAAM,UAAU,GAAG;IACvB,0CAA0C;IAC1C,GAAG,EAAE,MAAM,CAAA;IACX,kEAAkE;IAClE,UAAU,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;CAC3C,CAAA;AAED,wBAAgB,UAAU,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC,GAAG,EAAE,UAAU,KAAK,OAAO,CAAC,IAAI,CAAC;;;mBAA5B,UAAU,KAAK,OAAO,CAAC,IAAI,CAAC;EAExF;AAMD;;;GAGG;AACH,wBAAsB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAQrE;AAQD,qDAAqD;AACrD,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,GAAG,IAAI,CAKnE;AAED,iFAAiF;AACjF,wBAAsB,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAOhE"}
package/dist/hooks.js ADDED
@@ -0,0 +1,75 @@
1
+ var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExtension) || function (path, preserveJsx) {
2
+ if (typeof path === "string" && /^\.\.?\//.test(path)) {
3
+ return path.replace(/\.(tsx)$|((?:\.d)?)((?:\.[^./]+?)?)\.([cm]?)ts$/i, function (m, tsx, d, ext, cm) {
4
+ return tsx ? preserveJsx ? ".jsx" : ".js" : d && (!ext || !cm) ? m : (d + ext + "." + cm.toLowerCase() + "js");
5
+ });
6
+ }
7
+ return path;
8
+ };
9
+ /**
10
+ * Lifecycle hooks that run in response to server events.
11
+ *
12
+ * Place hook modules in the `hooks/` directory. Currently supported hooks:
13
+ *
14
+ * - `on-login.ts` — called after each successful OAuth login
15
+ *
16
+ * Each hook default-exports an async function that receives an event-specific
17
+ * context object.
18
+ *
19
+ * @example
20
+ * ```ts
21
+ * // hooks/on-login.ts
22
+ * import type { OnLoginCtx } from '@hatk/hatk/hooks'
23
+ *
24
+ * export default async function (ctx: OnLoginCtx) {
25
+ * // Ensure the user's repo is backfilled on first login
26
+ * await ctx.ensureRepo(ctx.did)
27
+ * }
28
+ * ```
29
+ */
30
+ import { existsSync } from 'node:fs';
31
+ import { resolve } from 'node:path';
32
+ import { log } from "./logger.js";
33
+ import { setRepoStatus } from "./database/db.js";
34
+ import { triggerAutoBackfill } from "./indexer.js";
35
+ export function defineHook(event, handler) {
36
+ return { __type: 'hook', event, handler };
37
+ }
38
+ let onLoginHook = null;
39
+ /**
40
+ * Discover and load the on-login hook from the project's `hooks/` directory.
41
+ * Looks for `on-login.ts` or `on-login.js`. Safe to call if no hook exists.
42
+ */
43
+ export async function loadOnLoginHook(hooksDir) {
44
+ const tsPath = resolve(hooksDir, 'on-login.ts');
45
+ const jsPath = resolve(hooksDir, 'on-login.js');
46
+ const path = existsSync(tsPath) ? tsPath : existsSync(jsPath) ? jsPath : null;
47
+ if (!path)
48
+ return;
49
+ const mod = await import(__rewriteRelativeImportExtension(/* @vite-ignore */ `${path}?t=${Date.now()}`));
50
+ onLoginHook = mod.default;
51
+ log('[hooks] on-login hook loaded');
52
+ }
53
+ /** Mark a DID as pending and trigger auto-backfill. */
54
+ async function ensureRepo(did) {
55
+ await setRepoStatus(did, 'pending');
56
+ triggerAutoBackfill(did);
57
+ }
58
+ /** Register a hook from a scanned server/ module. */
59
+ export function registerHook(event, handler) {
60
+ if (event === 'on-login') {
61
+ onLoginHook = handler;
62
+ log('[hooks] on-login hook registered');
63
+ }
64
+ }
65
+ /** Fire the on-login hook if loaded. Errors are logged but never block login. */
66
+ export async function fireOnLoginHook(did) {
67
+ if (!onLoginHook)
68
+ return;
69
+ try {
70
+ await onLoginHook({ did, ensureRepo });
71
+ }
72
+ catch (err) {
73
+ console.error('[hooks] onLogin hook error:', err.message);
74
+ }
75
+ }
package/dist/hydrate.js CHANGED
@@ -1,4 +1,4 @@
1
- import { getRecordsByUris, countByFieldBatch, lookupByFieldBatch, querySQL, reshapeRow, queryLabelsForUris, filterTakendownDids, } from "./db.js";
1
+ import { getRecordsByUris, countByFieldBatch, lookupByFieldBatch, querySQL, reshapeRow, queryLabelsForUris, filterTakendownDids, } from "./database/db.js";
2
2
  import { blobUrl } from "./xrpc.js";
3
3
  // --- Record Resolution ---
4
4
  /** Fetch records for URIs, reshape them, and filter out taken-down DIDs. */
package/dist/indexer.d.ts CHANGED
@@ -1,4 +1,13 @@
1
+ /**
2
+ * Auto-backfill a DID's repo when first seen on the firehose.
3
+ *
4
+ * Fetches the full repo via CAR export, inserts all records, then replays any
5
+ * firehose events that arrived during the backfill. Concurrency is capped at
6
+ * `maxConcurrentBackfills`. Failed backfills retry with exponential delay up
7
+ * to `maxRetries`.
8
+ */
1
9
  export declare function triggerAutoBackfill(did: string, attempt?: number): Promise<void>;
10
+ /** Configuration for the firehose indexer. */
2
11
  interface IndexerOpts {
3
12
  relayUrl: string;
4
13
  collections: Set<string>;
@@ -7,8 +16,19 @@ interface IndexerOpts {
7
16
  cursor?: string | null;
8
17
  fetchTimeout: number;
9
18
  maxRetries: number;
19
+ parallelism?: number;
10
20
  ftsRebuildInterval?: number;
11
21
  }
22
+ /**
23
+ * Connect to the AT Protocol relay firehose and begin indexing.
24
+ *
25
+ * Opens a WebSocket to `subscribeRepos`, processes commit messages synchronously
26
+ * on the event loop to minimize backpressure, and batches writes through
27
+ * {@link flushBuffer}. New DIDs trigger auto-backfill via {@link triggerAutoBackfill}.
28
+ * Reconnects automatically on disconnect after a 3s delay.
29
+ *
30
+ * @returns The WebSocket connection (for shutdown coordination)
31
+ */
12
32
  export declare function startIndexer(opts: IndexerOpts): Promise<WebSocket>;
13
33
  export {};
14
34
  //# sourceMappingURL=indexer.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../src/indexer.ts"],"names":[],"mappings":"AAkIA,wBAAsB,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAsDjF;AAED,UAAU,WAAW;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,iBAAiB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IAC/B,WAAW,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACzB,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACtB,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;IAClB,kBAAkB,CAAC,EAAE,MAAM,CAAA;CAC5B;AAyBD,wBAAsB,YAAY,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,SAAS,CAAC,CAkDxE"}
1
+ {"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../src/indexer.ts"],"names":[],"mappings":"AAuJA;;;;;;;GAOG;AACH,wBAAsB,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAgEjF;AAED,8CAA8C;AAC9C,UAAU,WAAW;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,iBAAiB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IAC/B,WAAW,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACzB,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACtB,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,kBAAkB,CAAC,EAAE,MAAM,CAAA;CAC5B;AAyBD;;;;;;;;;GASG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,SAAS,CAAC,CAmDxE"}
package/dist/indexer.js CHANGED
@@ -1,11 +1,11 @@
1
1
  import { cborDecode } from "./cbor.js";
2
2
  import { parseCarFrame } from "./car.js";
3
- import { insertRecord, deleteRecord, setCursor, setRepoStatus, getRepoRetryInfo, listAllRepoStatuses } from "./db.js";
3
+ import { insertRecord, deleteRecord, setCursor, setRepoStatus, getRepoRetryInfo, listAllRepoStatuses, getDatabasePort, } from "./database/db.js";
4
4
  import { backfillRepo } from "./backfill.js";
5
- import { rebuildAllIndexes } from "./fts.js";
5
+ import { rebuildAllIndexes } from "./database/fts.js";
6
6
  import { log, emit, timer } from "./logger.js";
7
7
  import { runLabelRules } from "./labels.js";
8
- import { getLexiconArray } from "./schema.js";
8
+ import { getLexiconArray } from "./database/schema.js";
9
9
  import { validateRecord } from '@bigmoves/lexicon';
10
10
  let buffer = [];
11
11
  let flushTimer = null;
@@ -18,7 +18,7 @@ let ftsRebuildInterval = 500;
18
18
  const pendingBuffers = new Map();
19
19
  // Track in-flight backfills to avoid duplicates
20
20
  const backfillInFlight = new Set();
21
- const MAX_CONCURRENT_BACKFILLS = 5;
21
+ const pendingReschedule = new Set();
22
22
  // In-memory cache of repo status to avoid flooding the DB read queue
23
23
  const repoStatusCache = new Map();
24
24
  // Set by startIndexer
@@ -27,6 +27,12 @@ let indexerSignalCollections;
27
27
  let indexerPinnedRepos = null;
28
28
  let indexerFetchTimeout;
29
29
  let indexerMaxRetries;
30
+ let maxConcurrentBackfills = 3;
31
+ /**
32
+ * Flush the write buffer — insert all buffered records, update the relay cursor,
33
+ * run label rules on inserted records, and trigger FTS rebuilds when the write
34
+ * threshold is reached. Emits a wide event with batch stats.
35
+ */
30
36
  async function flushBuffer() {
31
37
  if (buffer.length === 0)
32
38
  return;
@@ -86,9 +92,14 @@ async function flushBuffer() {
86
92
  writesSinceRebuild += batch.length;
87
93
  if (writesSinceRebuild >= ftsRebuildInterval) {
88
94
  writesSinceRebuild = 0;
89
- rebuildAllIndexes([...indexerCollections]).catch(() => { });
95
+ // Skip periodic full rebuild for SQLite — it uses incremental FTS updates
96
+ const port = getDatabasePort();
97
+ if (port.dialect !== 'sqlite') {
98
+ rebuildAllIndexes([...indexerCollections]).catch(() => { });
99
+ }
90
100
  }
91
101
  }
102
+ /** Schedule a flush after FLUSH_INTERVAL_MS if one isn't already pending. */
92
103
  function scheduleFlush() {
93
104
  if (flushTimer)
94
105
  return;
@@ -97,6 +108,7 @@ function scheduleFlush() {
97
108
  await flushBuffer();
98
109
  }, FLUSH_INTERVAL_MS);
99
110
  }
111
+ /** Add a record to the write buffer. Flushes immediately if BATCH_SIZE is reached. */
100
112
  function bufferWrite(item) {
101
113
  buffer.push(item);
102
114
  if (buffer.length >= BATCH_SIZE) {
@@ -110,9 +122,27 @@ function bufferWrite(item) {
110
122
  scheduleFlush();
111
123
  }
112
124
  }
125
+ /**
126
+ * Auto-backfill a DID's repo when first seen on the firehose.
127
+ *
128
+ * Fetches the full repo via CAR export, inserts all records, then replays any
129
+ * firehose events that arrived during the backfill. Concurrency is capped at
130
+ * `maxConcurrentBackfills`. Failed backfills retry with exponential delay up
131
+ * to `maxRetries`.
132
+ */
113
133
  export async function triggerAutoBackfill(did, attempt = 0) {
114
134
  if (backfillInFlight.has(did))
115
135
  return;
136
+ if (backfillInFlight.size >= maxConcurrentBackfills) {
137
+ if (!pendingReschedule.has(did)) {
138
+ pendingReschedule.add(did);
139
+ setTimeout(() => {
140
+ pendingReschedule.delete(did);
141
+ triggerAutoBackfill(did, attempt);
142
+ }, 10_000);
143
+ }
144
+ return;
145
+ }
116
146
  backfillInFlight.add(did);
117
147
  pendingBuffers.set(did, []);
118
148
  if (attempt === 0)
@@ -162,7 +192,7 @@ export async function triggerAutoBackfill(did, attempt = 0) {
162
192
  }, delayMs);
163
193
  }
164
194
  }
165
- // Periodic memory diagnostics
195
+ /** Emit a memory diagnostics wide event every 30s for observability. */
166
196
  function startMemoryDiagnostics() {
167
197
  setInterval(() => {
168
198
  const mem = process.memoryUsage();
@@ -184,6 +214,16 @@ function startMemoryDiagnostics() {
184
214
  });
185
215
  }, 30_000);
186
216
  }
217
+ /**
218
+ * Connect to the AT Protocol relay firehose and begin indexing.
219
+ *
220
+ * Opens a WebSocket to `subscribeRepos`, processes commit messages synchronously
221
+ * on the event loop to minimize backpressure, and batches writes through
222
+ * {@link flushBuffer}. New DIDs trigger auto-backfill via {@link triggerAutoBackfill}.
223
+ * Reconnects automatically on disconnect after a 3s delay.
224
+ *
225
+ * @returns The WebSocket connection (for shutdown coordination)
226
+ */
187
227
  export async function startIndexer(opts) {
188
228
  const { relayUrl, collections, cursor, fetchTimeout } = opts;
189
229
  if (opts.ftsRebuildInterval != null)
@@ -193,6 +233,7 @@ export async function startIndexer(opts) {
193
233
  indexerPinnedRepos = opts.pinnedRepos || null;
194
234
  indexerFetchTimeout = fetchTimeout;
195
235
  indexerMaxRetries = opts.maxRetries;
236
+ maxConcurrentBackfills = opts.parallelism ?? 3;
196
237
  // Pre-populate repo status cache from DB so non-signal updates
197
238
  // (e.g. profile changes) are processed for already-tracked DIDs
198
239
  if (repoStatusCache.size === 0) {
@@ -231,6 +272,11 @@ export async function startIndexer(opts) {
231
272
  });
232
273
  return ws;
233
274
  }
275
+ /**
276
+ * Process a single firehose message. Decodes the CBOR header/body, filters
277
+ * for relevant collections, validates records against lexicons, and routes
278
+ * writes to the buffer (or pending buffer if the DID is mid-backfill).
279
+ */
234
280
  function processMessage(bytes, collections) {
235
281
  const header = cborDecode(bytes, 0);
236
282
  const body = cborDecode(bytes, header.offset);
@@ -264,7 +310,7 @@ function processMessage(bytes, collections) {
264
310
  repoStatusCache.set(did, 'unknown');
265
311
  }
266
312
  if (hasSignalOp && (!indexerPinnedRepos || indexerPinnedRepos.has(did))) {
267
- if (repoStatus === null && backfillInFlight.size < MAX_CONCURRENT_BACKFILLS) {
313
+ if (repoStatus === null && backfillInFlight.size < maxConcurrentBackfills) {
268
314
  repoStatusCache.set(did, 'pending');
269
315
  triggerAutoBackfill(did);
270
316
  }
package/dist/labels.d.ts CHANGED
@@ -13,7 +13,34 @@ export interface LabelRuleContext {
13
13
  value: Record<string, any>;
14
14
  };
15
15
  }
16
+ export interface LabelModule {
17
+ definition?: LabelDefinition;
18
+ evaluate?: (ctx: LabelRuleContext) => Promise<string[]>;
19
+ }
20
+ export declare function defineLabels(module: LabelModule): {
21
+ definition?: LabelDefinition;
22
+ evaluate?: (ctx: LabelRuleContext) => Promise<string[]>;
23
+ __type: "labels";
24
+ };
25
+ /**
26
+ * Discover and load label rule modules from the `labels/` directory.
27
+ *
28
+ * Each module should default-export an object with an optional `definition`
29
+ * (label metadata like severity and blur behavior) and an optional `evaluate`
30
+ * function that returns label values to apply to a record.
31
+ *
32
+ * @param labelsDir - Absolute path to the `labels/` directory
33
+ */
16
34
  export declare function initLabels(labelsDir: string): Promise<void>;
35
+ /** Register a single label module from a scanned server/ module. */
36
+ export declare function registerLabelModule(name: string, labelMod: {
37
+ definition?: LabelDefinition;
38
+ evaluate?: (ctx: LabelRuleContext) => Promise<string[]>;
39
+ }): void;
40
+ /**
41
+ * Evaluate all loaded label rules against a record and persist any resulting labels.
42
+ * Called after each record is indexed. Rule errors are logged but never block indexing.
43
+ */
17
44
  export declare function runLabelRules(record: {
18
45
  uri: string;
19
46
  cid: string;
@@ -21,9 +48,16 @@ export declare function runLabelRules(record: {
21
48
  collection: string;
22
49
  value: Record<string, any>;
23
50
  }): Promise<void>;
51
+ /**
52
+ * Re-evaluate all label rules against every existing record in the given collections.
53
+ * Used by `/admin/rescan-labels` to apply new or updated rules retroactively.
54
+ *
55
+ * @returns Count of records scanned and new labels applied
56
+ */
24
57
  export declare function rescanLabels(collections: string[]): Promise<{
25
58
  scanned: number;
26
59
  labeled: number;
27
60
  }>;
61
+ /** Return all label definitions discovered during {@link initLabels}. */
28
62
  export declare function getLabelDefinitions(): LabelDefinition[];
29
63
  //# sourceMappingURL=labels.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"labels.d.ts","sourceRoot":"","sources":["../src/labels.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAIlD,wDAAwD;AACxD,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE;QACF,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,GAAG,EAAE,KAAK,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;QACtD,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,GAAG,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;KACtD,CAAA;IACD,MAAM,EAAE;QACN,GAAG,EAAE,MAAM,CAAA;QACX,GAAG,EAAE,MAAM,CAAA;QACX,GAAG,EAAE,MAAM,CAAA;QACX,UAAU,EAAE,MAAM,CAAA;QAClB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;KAC3B,CAAA;CACF;AAWD,wBAAsB,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAmCjE;AAED,wBAAsB,aAAa,CAAC,MAAM,EAAE;IAC1C,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,MAAM,CAAA;IAClB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;CAC3B,GAAG,OAAO,CAAC,IAAI,CAAC,CAyBhB;AAED,wBAAsB,YAAY,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,CAuCvG;AAED,wBAAgB,mBAAmB,IAAI,eAAe,EAAE,CAEvD"}
1
+ {"version":3,"file":"labels.d.ts","sourceRoot":"","sources":["../src/labels.ts"],"names":[],"mappings":"AA8BA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAIlD,wDAAwD;AACxD,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE;QACF,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,GAAG,EAAE,KAAK,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;QACtD,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,GAAG,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;KACtD,CAAA;IACD,MAAM,EAAE;QACN,GAAG,EAAE,MAAM,CAAA;QACX,GAAG,EAAE,MAAM,CAAA;QACX,GAAG,EAAE,MAAM,CAAA;QACX,UAAU,EAAE,MAAM,CAAA;QAClB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;KAC3B,CAAA;CACF;AAED,MAAM,WAAW,WAAW;IAC1B,UAAU,CAAC,EAAE,eAAe,CAAA;IAC5B,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE,gBAAgB,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;CACxD;AAED,wBAAgB,YAAY,CAAC,MAAM,EAAE,WAAW;iBAJjC,eAAe;eACjB,CAAC,GAAG,EAAE,gBAAgB,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC;;EAKxD;AAYD;;;;;;;;GAQG;AACH,wBAAsB,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAmCjE;AAED,oEAAoE;AACpE,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE;IAAE,UAAU,CAAC,EAAE,eAAe,CAAC;IAAC,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE,gBAAgB,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;CAAE,GAAG,IAAI,CAO3J;AAED;;;GAGG;AACH,wBAAsB,aAAa,CAAC,MAAM,EAAE;IAC1C,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,MAAM,CAAA;IAClB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;CAC3B,GAAG,OAAO,CAAC,IAAI,CAAC,CAyBhB;AAED;;;;;GAKG;AACH,wBAAsB,YAAY,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,CAuCvG;AAED,yEAAyE;AACzE,wBAAgB,mBAAmB,IAAI,eAAe,EAAE,CAEvD"}