@hatk/hatk 0.0.1-alpha.36 → 0.0.1-alpha.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backfill.js +6 -6
- package/dist/cli.js +56 -13
- package/dist/cloudflare/container.d.ts +73 -0
- package/dist/cloudflare/container.d.ts.map +1 -0
- package/dist/cloudflare/container.js +232 -0
- package/dist/cloudflare/hooks.d.ts +33 -0
- package/dist/cloudflare/hooks.d.ts.map +1 -0
- package/dist/cloudflare/hooks.js +40 -0
- package/dist/cloudflare/init.d.ts +27 -0
- package/dist/cloudflare/init.d.ts.map +1 -0
- package/dist/cloudflare/init.js +103 -0
- package/dist/cloudflare/worker.d.ts +27 -0
- package/dist/cloudflare/worker.d.ts.map +1 -0
- package/dist/cloudflare/worker.js +54 -0
- package/dist/database/adapters/d1.d.ts +56 -0
- package/dist/database/adapters/d1.d.ts.map +1 -0
- package/dist/database/adapters/d1.js +108 -0
- package/dist/database/db.d.ts +2 -0
- package/dist/database/db.d.ts.map +1 -1
- package/dist/database/db.js +18 -5
- package/dist/database/fts.js +1 -1
- package/dist/database/schema.d.ts +1 -0
- package/dist/database/schema.d.ts.map +1 -1
- package/dist/database/schema.js +27 -21
- package/dist/hydrate.d.ts.map +1 -1
- package/dist/hydrate.js +2 -13
- package/dist/labels.d.ts +1 -1
- package/dist/labels.d.ts.map +1 -1
- package/dist/labels.js +2 -2
- package/dist/opengraph.d.ts.map +1 -1
- package/dist/opengraph.js +3 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +7 -1
- package/dist/xrpc.d.ts +1 -0
- package/dist/xrpc.d.ts.map +1 -1
- package/dist/xrpc.js +3 -1
- package/package.json +1 -1
package/dist/backfill.js
CHANGED
|
@@ -200,6 +200,7 @@ export async function backfillRepo(did, collections, fetchTimeout) {
|
|
|
200
200
|
// Insert records in chunks to limit memory usage
|
|
201
201
|
const CHUNK_SIZE = 1000;
|
|
202
202
|
let chunk = [];
|
|
203
|
+
const validationSkips = {};
|
|
203
204
|
for (const entry of entries) {
|
|
204
205
|
const collection = entry.path.split('/')[0];
|
|
205
206
|
if (!collections.has(collection))
|
|
@@ -216,12 +217,7 @@ export async function backfillRepo(did, collections, fetchTimeout) {
|
|
|
216
217
|
const uri = `at://${did}/${collection}/${rkey}`;
|
|
217
218
|
const validationError = validateRecord(getLexiconArray(), collection, record);
|
|
218
219
|
if (validationError) {
|
|
219
|
-
|
|
220
|
-
uri,
|
|
221
|
-
collection,
|
|
222
|
-
path: validationError.path,
|
|
223
|
-
error: validationError.message,
|
|
224
|
-
});
|
|
220
|
+
validationSkips[collection] = (validationSkips[collection] || 0) + 1;
|
|
225
221
|
continue;
|
|
226
222
|
}
|
|
227
223
|
chunk.push({ collection, uri, cid: entry.cid, did, record });
|
|
@@ -242,6 +238,10 @@ export async function backfillRepo(did, collections, fetchTimeout) {
|
|
|
242
238
|
if (chunk.length > 0) {
|
|
243
239
|
count += await bulkInsertRecords(chunk);
|
|
244
240
|
}
|
|
241
|
+
const totalSkips = Object.values(validationSkips).reduce((a, b) => a + b, 0);
|
|
242
|
+
if (totalSkips > 0) {
|
|
243
|
+
emit('backfill', 'validation_skips', { did, total: totalSkips, by_collection: validationSkips });
|
|
244
|
+
}
|
|
245
245
|
await setRepoStatus(did, 'active', commit.rev, { handle });
|
|
246
246
|
return count;
|
|
247
247
|
}
|
package/dist/cli.js
CHANGED
|
@@ -1304,7 +1304,7 @@ else if (command === 'generate') {
|
|
|
1304
1304
|
out += `import type { ${[...usedWrappers].sort().join(', ')}, LexServerParams, Checked, Prettify, StrictArg } from '@hatk/hatk/lex-types'\n`;
|
|
1305
1305
|
out += `import type { XrpcContext } from '@hatk/hatk/xrpc'\n`;
|
|
1306
1306
|
out += `import { callXrpc as _callXrpc } from '@hatk/hatk/xrpc'\n`;
|
|
1307
|
-
out += `import { defineFeed as _defineFeed, type FeedResult, type FeedContext, type HydrateContext } from '@hatk/hatk/feeds'\n`;
|
|
1307
|
+
out += `import { defineFeed as _defineFeed, type FeedResult, type FeedContext, type HydrateContext, type Row } from '@hatk/hatk/feeds'\n`;
|
|
1308
1308
|
out += `import { seed as _seed, type SeedOpts } from '@hatk/hatk/seed'\n`;
|
|
1309
1309
|
// Emit ALL lexicons as `const ... = {...} as const` (including defs-only)
|
|
1310
1310
|
out += `\n// ─── Lexicon Definitions ────────────────────────────────────────────\n\n`;
|
|
@@ -1460,11 +1460,11 @@ else if (command === 'generate') {
|
|
|
1460
1460
|
out += `}\n`;
|
|
1461
1461
|
// Emit Ctx helper for typesafe XRPC handler contexts
|
|
1462
1462
|
out += `\n// ─── XRPC Helpers ───────────────────────────────────────────────────\n\n`;
|
|
1463
|
-
out += `export type { HydrateContext } from '@hatk/hatk/feeds'\n`;
|
|
1463
|
+
out += `export type { HydrateContext, Row } from '@hatk/hatk/feeds'\n`;
|
|
1464
1464
|
out += `export { InvalidRequestError, NotFoundError } from '@hatk/hatk/xrpc'\n`;
|
|
1465
1465
|
out += `export { defineSetup } from '@hatk/hatk/setup'\n`;
|
|
1466
1466
|
out += `export { defineHook } from '@hatk/hatk/hooks'\n`;
|
|
1467
|
-
out += `export {
|
|
1467
|
+
out += `export { defineLabel } from '@hatk/hatk/labels'\n`;
|
|
1468
1468
|
out += `export { defineOG } from '@hatk/hatk/opengraph'\n`;
|
|
1469
1469
|
out += `export { defineRenderer } from '@hatk/hatk/renderer'\n`;
|
|
1470
1470
|
out += `export type Ctx<K extends keyof XrpcSchema & keyof Registry> = XrpcContext<\n`;
|
|
@@ -1571,9 +1571,10 @@ else if (command === 'generate') {
|
|
|
1571
1571
|
clientOut += `export async function callXrpc<K extends keyof XrpcSchema & string>(\n`;
|
|
1572
1572
|
clientOut += ` nsid: K,\n`;
|
|
1573
1573
|
clientOut += ` arg?: CallArg<K>,\n`;
|
|
1574
|
+
clientOut += ` customFetch?: typeof globalThis.fetch,\n`;
|
|
1574
1575
|
clientOut += `): Promise<OutputOf<K>> {\n`;
|
|
1575
|
-
// Server-side bridge
|
|
1576
|
-
clientOut += ` if (typeof window === 'undefined') {\n`;
|
|
1576
|
+
// Server-side bridge (skip when customFetch is provided — let SvelteKit's fetch handle it)
|
|
1577
|
+
clientOut += ` if (typeof window === 'undefined' && !customFetch) {\n`;
|
|
1577
1578
|
clientOut += ` const bridge = (globalThis as any).__hatk_callXrpc\n`;
|
|
1578
1579
|
clientOut += ` if (!bridge) throw new Error('callXrpc: server bridge not available — is hatk initialized?')\n`;
|
|
1579
1580
|
if (procedureNsids.length > 0 || blobInputNsids.length > 0) {
|
|
@@ -1586,30 +1587,35 @@ else if (command === 'generate') {
|
|
|
1586
1587
|
}
|
|
1587
1588
|
clientOut += ` return bridge(nsid, arg) as Promise<OutputOf<K>>\n`;
|
|
1588
1589
|
clientOut += ` }\n`;
|
|
1589
|
-
// Client-side fetch
|
|
1590
|
-
clientOut += ` const
|
|
1590
|
+
// Client-side fetch (or server-side with customFetch for SSR deduplication)
|
|
1591
|
+
clientOut += ` const _fetch = customFetch ?? globalThis.fetch\n`;
|
|
1592
|
+
clientOut += ` // Use relative URL so SvelteKit's fetch can deduplicate server/client requests\n`;
|
|
1593
|
+
clientOut += ` let path = \`/xrpc/\${nsid}\`\n`;
|
|
1591
1594
|
if (blobInputNsids.length > 0) {
|
|
1592
1595
|
clientOut += ` if (_blobInputs.has(nsid)) {\n`;
|
|
1593
1596
|
clientOut += ` const blob = arg as Blob | ArrayBuffer\n`;
|
|
1594
1597
|
clientOut += ` const ct = blob instanceof Blob ? blob.type : 'application/octet-stream'\n`;
|
|
1595
|
-
clientOut += ` const res = await
|
|
1598
|
+
clientOut += ` const res = await _fetch(path, { method: 'POST', headers: { 'Content-Type': ct }, body: blob })\n`;
|
|
1596
1599
|
clientOut += ` if (!res.ok) throw new Error(\`XRPC \${nsid} failed: \${res.status}\`)\n`;
|
|
1597
1600
|
clientOut += ` return res.json() as Promise<OutputOf<K>>\n`;
|
|
1598
1601
|
clientOut += ` }\n`;
|
|
1599
1602
|
}
|
|
1600
1603
|
if (procedureNsids.length > 0) {
|
|
1601
1604
|
clientOut += ` if (_procedures.has(nsid)) {\n`;
|
|
1602
|
-
clientOut += ` const res = await
|
|
1603
|
-
clientOut += ` if (res.status === 401) { window.location.href = '/oauth/login'; return new Promise(() => {}) as any }\n`;
|
|
1605
|
+
clientOut += ` const res = await _fetch(path, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(arg) })\n`;
|
|
1606
|
+
clientOut += ` if (typeof window !== 'undefined' && res.status === 401) { const _h = getViewer()?.handle; window.location.href = _h ? \`/oauth/login?handle=\${encodeURIComponent(_h)}\` : '/oauth/login'; return new Promise(() => {}) as any }\n`;
|
|
1604
1607
|
clientOut += ` if (!res.ok) throw new Error(\`XRPC \${nsid} failed: \${res.status}\`)\n`;
|
|
1605
1608
|
clientOut += ` return res.json() as Promise<OutputOf<K>>\n`;
|
|
1606
1609
|
clientOut += ` }\n`;
|
|
1607
1610
|
}
|
|
1611
|
+
clientOut += ` const params = new URLSearchParams()\n`;
|
|
1608
1612
|
clientOut += ` for (const [k, v] of Object.entries(arg || {})) {\n`;
|
|
1609
|
-
clientOut += ` if (v != null)
|
|
1613
|
+
clientOut += ` if (v != null) params.set(k, String(v))\n`;
|
|
1610
1614
|
clientOut += ` }\n`;
|
|
1611
|
-
clientOut += ` const
|
|
1612
|
-
clientOut += ` if (
|
|
1615
|
+
clientOut += ` const qs = params.toString()\n`;
|
|
1616
|
+
clientOut += ` if (qs) path += \`?\${qs}\`\n`;
|
|
1617
|
+
clientOut += ` const res = await _fetch(path)\n`;
|
|
1618
|
+
clientOut += ` if (typeof window !== 'undefined' && res.status === 401) { window.location.href = '/oauth/login'; return new Promise(() => {}) as any }\n`;
|
|
1613
1619
|
clientOut += ` if (!res.ok) throw new Error(\`XRPC \${nsid} failed: \${res.status}\`)\n`;
|
|
1614
1620
|
clientOut += ` return res.json() as Promise<OutputOf<K>>\n`;
|
|
1615
1621
|
clientOut += `}\n`;
|
|
@@ -1617,6 +1623,43 @@ else if (command === 'generate') {
|
|
|
1617
1623
|
clientOut += `\nexport function getViewer(): { did: string; handle: string } | null {\n`;
|
|
1618
1624
|
clientOut += ` return (globalThis as any).__hatk_viewer ?? null\n`;
|
|
1619
1625
|
clientOut += `}\n`;
|
|
1626
|
+
// Auth helpers — login, logout, viewerDid
|
|
1627
|
+
clientOut += `\n// ─── Auth Helpers ────────────────────────────────────────────────────\n\n`;
|
|
1628
|
+
clientOut += `export async function login(handle: string): Promise<void> {\n`;
|
|
1629
|
+
clientOut += ` const res = await fetch(\`/oauth/login?handle=\${encodeURIComponent(handle)}\`, { redirect: 'manual' })\n`;
|
|
1630
|
+
clientOut += ` if (res.type === 'opaqueredirect') {\n`;
|
|
1631
|
+
clientOut += ` window.location.href = \`/oauth/login?handle=\${encodeURIComponent(handle)}\`\n`;
|
|
1632
|
+
clientOut += ` return\n`;
|
|
1633
|
+
clientOut += ` }\n`;
|
|
1634
|
+
clientOut += ` if (res.ok) return\n`;
|
|
1635
|
+
clientOut += ` const body = await res.json().catch(() => ({ error: 'Login failed' }))\n`;
|
|
1636
|
+
clientOut += ` throw new Error(body.error || 'Login failed')\n`;
|
|
1637
|
+
clientOut += `}\n\n`;
|
|
1638
|
+
clientOut += `export async function logout(): Promise<void> {\n`;
|
|
1639
|
+
clientOut += ` ;(globalThis as any).__hatk_viewer = null\n`;
|
|
1640
|
+
clientOut += ` await fetch('/auth/logout', { method: 'POST' }).catch(() => {})\n`;
|
|
1641
|
+
clientOut += `}\n\n`;
|
|
1642
|
+
clientOut += `export function viewerDid(): string | null {\n`;
|
|
1643
|
+
clientOut += ` if (typeof window === 'undefined') return null\n`;
|
|
1644
|
+
clientOut += ` const viewer = (globalThis as any).__hatk_viewer\n`;
|
|
1645
|
+
clientOut += ` return viewer?.did ?? null\n`;
|
|
1646
|
+
clientOut += `}\n\n`;
|
|
1647
|
+
clientOut += `// Expose viewer for getViewer() bridge\n`;
|
|
1648
|
+
clientOut += `;(globalThis as any).__hatk_auth = { viewerDid }\n`;
|
|
1649
|
+
// parseViewer — server-side session cookie resolution for +layout.server.ts
|
|
1650
|
+
clientOut += `\n// ─── Server Helpers ──────────────────────────────────────────────────\n\n`;
|
|
1651
|
+
clientOut += `export async function parseViewer(cookies: { get(name: string): string | undefined }): Promise<{ did: string; handle?: string } | null> {\n`;
|
|
1652
|
+
clientOut += ` const parseSessionCookie = (globalThis as any).__hatk_parseSessionCookie\n`;
|
|
1653
|
+
clientOut += ` if (!parseSessionCookie) return null\n`;
|
|
1654
|
+
clientOut += ` const cookieValue = cookies.get('__hatk_session')\n`;
|
|
1655
|
+
clientOut += ` if (!cookieValue) return null\n`;
|
|
1656
|
+
clientOut += ` try {\n`;
|
|
1657
|
+
clientOut += ` const request = new Request('http://localhost', { headers: { cookie: \`__hatk_session=\${cookieValue}\` } })\n`;
|
|
1658
|
+
clientOut += ` const viewer = await parseSessionCookie(request)\n`;
|
|
1659
|
+
clientOut += ` if (viewer) (globalThis as any).__hatk_viewer = viewer\n`;
|
|
1660
|
+
clientOut += ` return viewer\n`;
|
|
1661
|
+
clientOut += ` } catch { return null }\n`;
|
|
1662
|
+
clientOut += `}\n`;
|
|
1620
1663
|
writeFileSync('./hatk.generated.client.ts', clientOut);
|
|
1621
1664
|
console.log(`Generated ${outPath} with ${entries.length} types: ${entries.map((e) => capitalize(varNames.get(e.nsid))).join(', ')}`);
|
|
1622
1665
|
console.log(`Generated ./hatk.generated.client.ts (client-safe subset)`);
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cloudflare Container entry point for hatk.
|
|
3
|
+
*
|
|
4
|
+
* Runs as a long-lived Node.js process alongside the Worker. Handles the
|
|
5
|
+
* firehose indexer and backfill loop — the Worker delegates resync requests
|
|
6
|
+
* here via RPC (Cloudflare Container service bindings).
|
|
7
|
+
*
|
|
8
|
+
* No HTTP server — all communication is via the Container RPC interface.
|
|
9
|
+
*/
|
|
10
|
+
export interface Env {
|
|
11
|
+
/** Cloudflare D1 database binding */
|
|
12
|
+
DB: D1Database;
|
|
13
|
+
HATK_RELAY: string;
|
|
14
|
+
HATK_PLC: string;
|
|
15
|
+
HATK_OAUTH_ISSUER?: string;
|
|
16
|
+
HATK_OAUTH_SCOPES?: string;
|
|
17
|
+
HATK_ADMINS?: string;
|
|
18
|
+
HATK_COLLECTIONS?: string;
|
|
19
|
+
HATK_BACKFILL_PARALLELISM?: string;
|
|
20
|
+
HATK_BACKFILL_FETCH_TIMEOUT?: string;
|
|
21
|
+
HATK_BACKFILL_MAX_RETRIES?: string;
|
|
22
|
+
HATK_BACKFILL_FULL_NETWORK?: string;
|
|
23
|
+
HATK_BACKFILL_REPOS?: string;
|
|
24
|
+
HATK_BACKFILL_SIGNAL_COLLECTIONS?: string;
|
|
25
|
+
}
|
|
26
|
+
interface D1Database {
|
|
27
|
+
prepare(sql: string): any;
|
|
28
|
+
batch<T = unknown>(statements: any[]): Promise<any[]>;
|
|
29
|
+
exec(sql: string): Promise<any>;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Resync a single DID by triggering auto-backfill.
|
|
33
|
+
* Called by the Worker via Container service binding RPC.
|
|
34
|
+
*/
|
|
35
|
+
declare function resync(did: string): Promise<void>;
|
|
36
|
+
/**
|
|
37
|
+
* Trigger a full re-enumeration backfill of all repos.
|
|
38
|
+
* Called by the Worker via Container service binding RPC.
|
|
39
|
+
*/
|
|
40
|
+
declare function resyncAll(): Promise<void>;
|
|
41
|
+
/**
|
|
42
|
+
* Return basic status info about the Container.
|
|
43
|
+
* Called by the Worker for health checks / admin UI.
|
|
44
|
+
*/
|
|
45
|
+
declare function getStatus(): {
|
|
46
|
+
initialized: boolean;
|
|
47
|
+
collections: string[];
|
|
48
|
+
uptimeMs: number;
|
|
49
|
+
};
|
|
50
|
+
/**
|
|
51
|
+
* Cloudflare Container entry point.
|
|
52
|
+
*
|
|
53
|
+
* Containers expose RPC methods that the Worker can call via the service binding.
|
|
54
|
+
* The Container also handles fetch requests routed from the Worker, but for hatk
|
|
55
|
+
* all Worker-to-Container communication uses the RPC methods above.
|
|
56
|
+
*/
|
|
57
|
+
declare const _default: {
|
|
58
|
+
/**
|
|
59
|
+
* Container startup — called when the Container is first instantiated.
|
|
60
|
+
* Initializes the database, starts the firehose, and begins backfill.
|
|
61
|
+
*/
|
|
62
|
+
start(env: Env): Promise<void>;
|
|
63
|
+
/**
|
|
64
|
+
* Handle fetch requests forwarded from the Worker.
|
|
65
|
+
* The Container doesn't serve HTTP — return 404 for any direct requests.
|
|
66
|
+
*/
|
|
67
|
+
fetch(request: Request, env: Env): Promise<Response>;
|
|
68
|
+
resync: typeof resync;
|
|
69
|
+
resyncAll: typeof resyncAll;
|
|
70
|
+
getStatus: typeof getStatus;
|
|
71
|
+
};
|
|
72
|
+
export default _default;
|
|
73
|
+
//# sourceMappingURL=container.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"container.d.ts","sourceRoot":"","sources":["../../src/cloudflare/container.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AA0BH,MAAM,WAAW,GAAG;IAClB,qCAAqC;IACrC,EAAE,EAAE,UAAU,CAAA;IAGd,UAAU,EAAE,MAAM,CAAA;IAClB,QAAQ,EAAE,MAAM,CAAA;IAChB,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAA;IAGzB,yBAAyB,CAAC,EAAE,MAAM,CAAA;IAClC,2BAA2B,CAAC,EAAE,MAAM,CAAA;IACpC,yBAAyB,CAAC,EAAE,MAAM,CAAA;IAClC,0BAA0B,CAAC,EAAE,MAAM,CAAA;IACnC,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B,gCAAgC,CAAC,EAAE,MAAM,CAAA;CAC1C;AAGD,UAAU,UAAU;IAClB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;IACzB,KAAK,CAAC,CAAC,GAAG,OAAO,EAAE,UAAU,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;IACrD,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAA;CAChC;AAoLD;;;GAGG;AACH,iBAAe,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAEhD;AAED;;;GAGG;AACH,iBAAe,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC,CAExC;AAED;;;GAGG;AACH,iBAAS,SAAS,IAAI;IAAE,WAAW,EAAE,OAAO,CAAC;IAAC,WAAW,EAAE,MAAM,EAAE,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAMtF;AAID;;;;;;GAMG;;IAED;;;OAGG;eACc,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;IAIpC;;;OAGG;mBACkB,OAAO,OAAO,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC;;;;;AAb5D,wBA0BC"}
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cloudflare Container entry point for hatk.
|
|
3
|
+
*
|
|
4
|
+
* Runs as a long-lived Node.js process alongside the Worker. Handles the
|
|
5
|
+
* firehose indexer and backfill loop — the Worker delegates resync requests
|
|
6
|
+
* here via RPC (Cloudflare Container service bindings).
|
|
7
|
+
*
|
|
8
|
+
* No HTTP server — all communication is via the Container RPC interface.
|
|
9
|
+
*/
|
|
10
|
+
import { D1Adapter } from "../database/adapters/d1.js";
|
|
11
|
+
import { initDatabase, getCursor, migrateSchema } from "../database/db.js";
|
|
12
|
+
import { storeLexicons, discoverCollections, buildSchemas } from "../database/schema.js";
|
|
13
|
+
import { discoverViews } from "../views.js";
|
|
14
|
+
import { getDialect } from "../database/dialect.js";
|
|
15
|
+
import { setSearchPort } from "../database/fts.js";
|
|
16
|
+
import { rebuildAllIndexes } from "../database/fts.js";
|
|
17
|
+
import { registerCoreHandlers } from "../server.js";
|
|
18
|
+
import { configureRelay } from "../xrpc.js";
|
|
19
|
+
import { startIndexer, triggerAutoBackfill } from "../indexer.js";
|
|
20
|
+
import { runBackfill } from "../backfill.js";
|
|
21
|
+
import { relayHttpUrl } from "../config.js";
|
|
22
|
+
import { validateLexicons } from '@bigmoves/lexicon';
|
|
23
|
+
import { log } from "../logger.js";
|
|
24
|
+
// ---------- Container state ----------
|
|
25
|
+
let initialized = false;
|
|
26
|
+
let initPromise = null;
|
|
27
|
+
let collections = [];
|
|
28
|
+
let collectionSet = new Set();
|
|
29
|
+
let backfillOpts = null;
|
|
30
|
+
let startedAt = 0;
|
|
31
|
+
/**
|
|
32
|
+
* One-time initialization. Mirrors main.ts startup minus the HTTP server.
|
|
33
|
+
*/
|
|
34
|
+
async function initialize(env) {
|
|
35
|
+
startedAt = Date.now();
|
|
36
|
+
// 1. Parse config from env vars
|
|
37
|
+
const relay = env.HATK_RELAY || 'wss://bsky.network';
|
|
38
|
+
const plc = env.HATK_PLC || 'https://plc.directory';
|
|
39
|
+
configureRelay(relay);
|
|
40
|
+
const admins = env.HATK_ADMINS ? env.HATK_ADMINS.split(',').map((s) => s.trim()) : [];
|
|
41
|
+
// 2. Load lexicons — injected at build time via virtual module
|
|
42
|
+
let lexicons;
|
|
43
|
+
try {
|
|
44
|
+
// @ts-expect-error — virtual module generated at build time
|
|
45
|
+
const lexiconModule = await import('virtual:hatk-lexicons');
|
|
46
|
+
lexicons = new Map(Object.entries(lexiconModule.default));
|
|
47
|
+
}
|
|
48
|
+
catch {
|
|
49
|
+
lexicons = new Map();
|
|
50
|
+
}
|
|
51
|
+
const lexiconErrors = validateLexicons([...lexicons.values()]);
|
|
52
|
+
if (lexiconErrors) {
|
|
53
|
+
for (const [nsid, errors] of Object.entries(lexiconErrors)) {
|
|
54
|
+
for (const err of errors) {
|
|
55
|
+
console.error(`[container] Invalid lexicon ${nsid}: ${err}`);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
throw new Error('Invalid lexicons — check build output');
|
|
59
|
+
}
|
|
60
|
+
storeLexicons(lexicons);
|
|
61
|
+
// 3. Determine collections
|
|
62
|
+
collections = env.HATK_COLLECTIONS
|
|
63
|
+
? env.HATK_COLLECTIONS.split(',').map((s) => s.trim())
|
|
64
|
+
: discoverCollections(lexicons);
|
|
65
|
+
collectionSet = new Set(collections);
|
|
66
|
+
if (collections.length === 0) {
|
|
67
|
+
log('[container] No record collections found — running in indexer-only mode');
|
|
68
|
+
}
|
|
69
|
+
log(`[container] Loaded config: ${collections.length} collections`);
|
|
70
|
+
// 4. Build schemas and init D1
|
|
71
|
+
discoverViews();
|
|
72
|
+
const engineDialect = getDialect('d1');
|
|
73
|
+
const { schemas, ddlStatements } = buildSchemas(lexicons, collections, engineDialect);
|
|
74
|
+
const adapter = new D1Adapter();
|
|
75
|
+
adapter.initWithBinding(env.DB);
|
|
76
|
+
setSearchPort(null); // D1 uses SQLite FTS natively
|
|
77
|
+
await initDatabase(adapter, ':memory:', schemas, ddlStatements);
|
|
78
|
+
// Auto-migrate schema if lexicons changed
|
|
79
|
+
const migrationChanges = await migrateSchema(schemas);
|
|
80
|
+
if (migrationChanges.length > 0) {
|
|
81
|
+
log(`[container] Applied ${migrationChanges.length} schema migration(s)`);
|
|
82
|
+
}
|
|
83
|
+
// 5. Init server directory handlers (feeds, labels, hooks, xrpc, setup)
|
|
84
|
+
// In Containers, we load these via the bundled virtual module like the Worker.
|
|
85
|
+
// The server/ directory scanning won't work in a Container since there's no filesystem
|
|
86
|
+
// layout matching the dev project. For now, register core handlers only.
|
|
87
|
+
// When build tooling (Task 7) bundles server handlers, they'll be imported here.
|
|
88
|
+
const oauthConfig = env.HATK_OAUTH_ISSUER
|
|
89
|
+
? {
|
|
90
|
+
issuer: env.HATK_OAUTH_ISSUER,
|
|
91
|
+
scopes: env.HATK_OAUTH_SCOPES ? env.HATK_OAUTH_SCOPES.split(',').map((s) => s.trim()) : ['read', 'write'],
|
|
92
|
+
clients: [],
|
|
93
|
+
}
|
|
94
|
+
: null;
|
|
95
|
+
registerCoreHandlers(collections, oauthConfig);
|
|
96
|
+
// 6. Parse backfill config
|
|
97
|
+
const backfillConfig = {
|
|
98
|
+
fullNetwork: env.HATK_BACKFILL_FULL_NETWORK === 'true',
|
|
99
|
+
parallelism: env.HATK_BACKFILL_PARALLELISM ? parseInt(env.HATK_BACKFILL_PARALLELISM, 10) : 10,
|
|
100
|
+
fetchTimeout: env.HATK_BACKFILL_FETCH_TIMEOUT ? parseInt(env.HATK_BACKFILL_FETCH_TIMEOUT, 10) : 30,
|
|
101
|
+
maxRetries: env.HATK_BACKFILL_MAX_RETRIES ? parseInt(env.HATK_BACKFILL_MAX_RETRIES, 10) : 5,
|
|
102
|
+
repos: env.HATK_BACKFILL_REPOS ? env.HATK_BACKFILL_REPOS.split(',').map((s) => s.trim()) : undefined,
|
|
103
|
+
signalCollections: env.HATK_BACKFILL_SIGNAL_COLLECTIONS
|
|
104
|
+
? env.HATK_BACKFILL_SIGNAL_COLLECTIONS.split(',').map((s) => s.trim())
|
|
105
|
+
: undefined,
|
|
106
|
+
};
|
|
107
|
+
backfillOpts = {
|
|
108
|
+
pdsUrl: relayHttpUrl(relay),
|
|
109
|
+
plcUrl: plc,
|
|
110
|
+
collections: collectionSet,
|
|
111
|
+
config: backfillConfig,
|
|
112
|
+
};
|
|
113
|
+
// 7. Start firehose indexer
|
|
114
|
+
const cursor = await getCursor('relay');
|
|
115
|
+
startIndexer({
|
|
116
|
+
relayUrl: relay,
|
|
117
|
+
collections: collectionSet,
|
|
118
|
+
signalCollections: backfillConfig.signalCollections ? new Set(backfillConfig.signalCollections) : undefined,
|
|
119
|
+
pinnedRepos: backfillConfig.repos ? new Set(backfillConfig.repos) : undefined,
|
|
120
|
+
cursor,
|
|
121
|
+
fetchTimeout: backfillConfig.fetchTimeout,
|
|
122
|
+
maxRetries: backfillConfig.maxRetries,
|
|
123
|
+
parallelism: backfillConfig.parallelism,
|
|
124
|
+
});
|
|
125
|
+
log('[container] Firehose indexer started');
|
|
126
|
+
// 8. Run backfill in background
|
|
127
|
+
runBackfillAndRestart();
|
|
128
|
+
initialized = true;
|
|
129
|
+
log('[container] Initialization complete');
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Run backfill, rebuild FTS indexes, and restart the process if records
|
|
133
|
+
* were imported (to reclaim memory from CAR parsing). Mirrors main.ts behavior.
|
|
134
|
+
*/
|
|
135
|
+
function runBackfillAndRestart() {
|
|
136
|
+
if (!backfillOpts)
|
|
137
|
+
return;
|
|
138
|
+
runBackfill(backfillOpts)
|
|
139
|
+
.then(async (recordCount) => {
|
|
140
|
+
log('[container] Backfill complete, building FTS indexes...');
|
|
141
|
+
await rebuildAllIndexes(collections);
|
|
142
|
+
log('[container] FTS indexes ready');
|
|
143
|
+
return recordCount;
|
|
144
|
+
})
|
|
145
|
+
.then((recordCount) => {
|
|
146
|
+
if (recordCount > 0) {
|
|
147
|
+
log('[container] Restarting to reclaim memory...');
|
|
148
|
+
process.exit(1);
|
|
149
|
+
}
|
|
150
|
+
})
|
|
151
|
+
.catch((err) => {
|
|
152
|
+
console.error('[container] Backfill error:', err.message);
|
|
153
|
+
});
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* Ensure initialization has completed. Uses a shared promise so concurrent
|
|
157
|
+
* RPC calls don't trigger multiple inits.
|
|
158
|
+
*/
|
|
159
|
+
function ensureInit(env) {
|
|
160
|
+
if (initialized)
|
|
161
|
+
return Promise.resolve();
|
|
162
|
+
if (!initPromise) {
|
|
163
|
+
initPromise = initialize(env).catch((err) => {
|
|
164
|
+
initPromise = null;
|
|
165
|
+
throw err;
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
return initPromise;
|
|
169
|
+
}
|
|
170
|
+
// ---------- RPC methods ----------
|
|
171
|
+
/**
|
|
172
|
+
* Resync a single DID by triggering auto-backfill.
|
|
173
|
+
* Called by the Worker via Container service binding RPC.
|
|
174
|
+
*/
|
|
175
|
+
async function resync(did) {
|
|
176
|
+
await triggerAutoBackfill(did);
|
|
177
|
+
}
|
|
178
|
+
/**
|
|
179
|
+
* Trigger a full re-enumeration backfill of all repos.
|
|
180
|
+
* Called by the Worker via Container service binding RPC.
|
|
181
|
+
*/
|
|
182
|
+
async function resyncAll() {
|
|
183
|
+
runBackfillAndRestart();
|
|
184
|
+
}
|
|
185
|
+
/**
|
|
186
|
+
* Return basic status info about the Container.
|
|
187
|
+
* Called by the Worker for health checks / admin UI.
|
|
188
|
+
*/
|
|
189
|
+
function getStatus() {
|
|
190
|
+
return {
|
|
191
|
+
initialized,
|
|
192
|
+
collections,
|
|
193
|
+
uptimeMs: startedAt > 0 ? Date.now() - startedAt : 0,
|
|
194
|
+
};
|
|
195
|
+
}
|
|
196
|
+
// ---------- Container export ----------
|
|
197
|
+
/**
|
|
198
|
+
* Cloudflare Container entry point.
|
|
199
|
+
*
|
|
200
|
+
* Containers expose RPC methods that the Worker can call via the service binding.
|
|
201
|
+
* The Container also handles fetch requests routed from the Worker, but for hatk
|
|
202
|
+
* all Worker-to-Container communication uses the RPC methods above.
|
|
203
|
+
*/
|
|
204
|
+
export default {
|
|
205
|
+
/**
|
|
206
|
+
* Container startup — called when the Container is first instantiated.
|
|
207
|
+
* Initializes the database, starts the firehose, and begins backfill.
|
|
208
|
+
*/
|
|
209
|
+
async start(env) {
|
|
210
|
+
await ensureInit(env);
|
|
211
|
+
},
|
|
212
|
+
/**
|
|
213
|
+
* Handle fetch requests forwarded from the Worker.
|
|
214
|
+
* The Container doesn't serve HTTP — return 404 for any direct requests.
|
|
215
|
+
*/
|
|
216
|
+
async fetch(request, env) {
|
|
217
|
+
await ensureInit(env);
|
|
218
|
+
return new Response(JSON.stringify({ error: 'Container does not serve HTTP requests' }), {
|
|
219
|
+
status: 404,
|
|
220
|
+
headers: { 'Content-Type': 'application/json' },
|
|
221
|
+
});
|
|
222
|
+
},
|
|
223
|
+
// RPC methods exposed to the Worker via service binding
|
|
224
|
+
resync,
|
|
225
|
+
resyncAll,
|
|
226
|
+
getStatus,
|
|
227
|
+
};
|
|
228
|
+
// Graceful shutdown
|
|
229
|
+
process.on('SIGTERM', () => {
|
|
230
|
+
log('[container] Received SIGTERM, shutting down...');
|
|
231
|
+
process.exit(0);
|
|
232
|
+
});
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SvelteKit handle hook for Cloudflare Workers.
|
|
3
|
+
*
|
|
4
|
+
* Use with @sveltejs/adapter-cloudflare. Lazily initializes hatk on the first
|
|
5
|
+
* request using the D1 binding from platform.env, then intercepts hatk API
|
|
6
|
+
* routes before SvelteKit processes them.
|
|
7
|
+
*
|
|
8
|
+
* @example
|
|
9
|
+
* ```ts
|
|
10
|
+
* // app/hooks.server.ts
|
|
11
|
+
* import { createHandle } from '@hatk/hatk/cloudflare/hooks'
|
|
12
|
+
* export const handle = createHandle()
|
|
13
|
+
* ```
|
|
14
|
+
*/
|
|
15
|
+
type MaybePromise<T> = T | Promise<T>;
|
|
16
|
+
/** Minimal SvelteKit Handle type to avoid depending on @sveltejs/kit. */
|
|
17
|
+
type Handle = (input: {
|
|
18
|
+
event: {
|
|
19
|
+
request: Request;
|
|
20
|
+
url: URL;
|
|
21
|
+
platform?: {
|
|
22
|
+
env?: Record<string, unknown>;
|
|
23
|
+
};
|
|
24
|
+
};
|
|
25
|
+
resolve: (event: any) => MaybePromise<Response>;
|
|
26
|
+
}) => MaybePromise<Response>;
|
|
27
|
+
/**
|
|
28
|
+
* Create a SvelteKit `handle` function that initializes hatk with D1
|
|
29
|
+
* and intercepts API routes.
|
|
30
|
+
*/
|
|
31
|
+
export declare function createHandle(): Handle;
|
|
32
|
+
export {};
|
|
33
|
+
//# sourceMappingURL=hooks.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hooks.d.ts","sourceRoot":"","sources":["../../src/cloudflare/hooks.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAKH,KAAK,YAAY,CAAC,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;AAErC,yEAAyE;AACzE,KAAK,MAAM,GAAG,CAAC,KAAK,EAAE;IACpB,KAAK,EAAE;QACL,OAAO,EAAE,OAAO,CAAA;QAChB,GAAG,EAAE,GAAG,CAAA;QACR,QAAQ,CAAC,EAAE;YAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;SAAE,CAAA;KAC7C,CAAA;IACD,OAAO,EAAE,CAAC,KAAK,EAAE,GAAG,KAAK,YAAY,CAAC,QAAQ,CAAC,CAAA;CAChD,KAAK,YAAY,CAAC,QAAQ,CAAC,CAAA;AAE5B;;;GAGG;AACH,wBAAgB,YAAY,IAAI,MAAM,CAsBrC"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SvelteKit handle hook for Cloudflare Workers.
|
|
3
|
+
*
|
|
4
|
+
* Use with @sveltejs/adapter-cloudflare. Lazily initializes hatk on the first
|
|
5
|
+
* request using the D1 binding from platform.env, then intercepts hatk API
|
|
6
|
+
* routes before SvelteKit processes them.
|
|
7
|
+
*
|
|
8
|
+
* @example
|
|
9
|
+
* ```ts
|
|
10
|
+
* // app/hooks.server.ts
|
|
11
|
+
* import { createHandle } from '@hatk/hatk/cloudflare/hooks'
|
|
12
|
+
* export const handle = createHandle()
|
|
13
|
+
* ```
|
|
14
|
+
*/
|
|
15
|
+
import { ensureInit, getHandler } from "./init.js";
|
|
16
|
+
import { isHatkRoute } from "../adapter.js";
|
|
17
|
+
/**
|
|
18
|
+
* Create a SvelteKit `handle` function that initializes hatk with D1
|
|
19
|
+
* and intercepts API routes.
|
|
20
|
+
*/
|
|
21
|
+
export function createHandle() {
|
|
22
|
+
return async ({ event, resolve }) => {
|
|
23
|
+
const env = event.platform?.env;
|
|
24
|
+
if (!env || !env.DB) {
|
|
25
|
+
// Not running on Cloudflare (e.g. dev mode) — pass through
|
|
26
|
+
return resolve(event);
|
|
27
|
+
}
|
|
28
|
+
// Lazy init hatk with the D1 binding
|
|
29
|
+
await ensureInit(env);
|
|
30
|
+
// hatk API routes
|
|
31
|
+
if (isHatkRoute(event.url.pathname)) {
|
|
32
|
+
const handler = getHandler();
|
|
33
|
+
if (handler) {
|
|
34
|
+
return handler(event.request);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
// Everything else → SvelteKit
|
|
38
|
+
return resolve(event);
|
|
39
|
+
};
|
|
40
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared Cloudflare initialization logic used by both the standalone Worker
|
|
3
|
+
* entry and the SvelteKit handle hook.
|
|
4
|
+
*/
|
|
5
|
+
export interface CloudflareEnv {
|
|
6
|
+
DB: D1Database;
|
|
7
|
+
HATK_RELAY?: string;
|
|
8
|
+
HATK_PLC?: string;
|
|
9
|
+
HATK_OAUTH_ISSUER?: string;
|
|
10
|
+
HATK_OAUTH_SCOPES?: string;
|
|
11
|
+
HATK_ADMINS?: string;
|
|
12
|
+
HATK_COLLECTIONS?: string;
|
|
13
|
+
[key: string]: unknown;
|
|
14
|
+
}
|
|
15
|
+
interface D1Database {
|
|
16
|
+
prepare(sql: string): any;
|
|
17
|
+
batch<T = unknown>(statements: any[]): Promise<any[]>;
|
|
18
|
+
exec(sql: string): Promise<any>;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Ensure initialization has completed. Concurrent calls share the same promise.
|
|
22
|
+
*/
|
|
23
|
+
export declare function ensureInit(env: CloudflareEnv): Promise<void>;
|
|
24
|
+
/** Get the hatk request handler (only valid after init). */
|
|
25
|
+
export declare function getHandler(): ((request: Request) => Promise<Response>) | null;
|
|
26
|
+
export {};
|
|
27
|
+
//# sourceMappingURL=init.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../../src/cloudflare/init.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAgBH,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,UAAU,CAAA;IACd,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CACvB;AAED,UAAU,UAAU;IAClB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;IACzB,KAAK,CAAC,CAAC,GAAG,OAAO,EAAE,UAAU,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;IACrD,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAA;CAChC;AAoFD;;GAEG;AACH,wBAAgB,UAAU,CAAC,GAAG,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,CAS5D;AAED,4DAA4D;AAC5D,wBAAgB,UAAU,IAAI,CAAC,CAAC,OAAO,EAAE,OAAO,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAC,GAAG,IAAI,CAE7E"}
|