@hatk/hatk 0.0.1-alpha.93 → 0.0.1-alpha.94
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/indexer.d.ts +11 -1
- package/dist/indexer.d.ts.map +1 -1
- package/dist/indexer.js +16 -2
- package/dist/jetstream.d.ts.map +1 -1
- package/dist/jetstream.js +2 -2
- package/dist/main.js +2 -2
- package/package.json +1 -1
package/dist/indexer.d.ts
CHANGED
|
@@ -23,7 +23,7 @@ export declare function getLastSeq(): number | null;
|
|
|
23
23
|
*/
|
|
24
24
|
export declare function checkpointCursor(): Promise<void>;
|
|
25
25
|
/** Point cursor persistence at a stream's own `_cursor` row. See {@link cursorKey}. */
|
|
26
|
-
export declare function setCursorKey(key:
|
|
26
|
+
export declare function setCursorKey(key: string): void;
|
|
27
27
|
/**
|
|
28
28
|
* The cursor a reconnect should resume from: the latest seq this process has
|
|
29
29
|
* seen, falling back to the boot-time cursor before any message has arrived.
|
|
@@ -123,6 +123,16 @@ export interface AuxIndexerOpts {
|
|
|
123
123
|
}
|
|
124
124
|
/** The `_cursor` row an auxiliary firehose persists its position to. */
|
|
125
125
|
export declare function auxCursorKey(relayUrl: string): string;
|
|
126
|
+
/**
|
|
127
|
+
* The `_cursor` row a Jetstream instance persists its position to.
|
|
128
|
+
*
|
|
129
|
+
* Named for the instance, because a Jetstream sequence means nothing on
|
|
130
|
+
* another one. Pointed at a second instance — a region swap, an outage
|
|
131
|
+
* worked around — a shared row would hand over a number from somewhere else,
|
|
132
|
+
* and the stream would resume at whatever that happens to address there:
|
|
133
|
+
* no error, no refusal, just a silent jump past whatever fell in between.
|
|
134
|
+
*/
|
|
135
|
+
export declare function jetstreamCursorKey(jetstreamUrl: string): string;
|
|
126
136
|
/**
|
|
127
137
|
* Tail a second `subscribeRepos` alongside the primary stream.
|
|
128
138
|
*
|
package/dist/indexer.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../src/indexer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;AA+ChD,iFAAiF;AACjF,MAAM,WAAW,QAAQ;IACvB,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAA;IACtC,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,yBAAyB;IACzB,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,yBAAyB;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;CAC7B;
|
|
1
|
+
{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../src/indexer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;AA+ChD,iFAAiF;AACjF,MAAM,WAAW,QAAQ;IACvB,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAA;IACtC,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,yBAAyB;IACzB,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,yBAAyB;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;CAC7B;AAkMD,6GAA6G;AAC7G,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAEzC;AAED,6EAA6E;AAC7E,wBAAgB,UAAU,IAAI,MAAM,GAAG,IAAI,CAE1C;AAED;;;;;;;GAOG;AACH,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC,CAStD;AAED,uFAAuF;AACvF,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAE9C;AAED;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,MAAM,GAAG,IAAI,GAAG,SAAS,CAErH;AAED,+CAA+C;AAC/C,wBAAgB,yBAAyB,IAAI,IAAI,CAIhD;AAED;;;;;;;;;GASG;AACH,wBAAgB,0BAA0B,IAAI,IAAI,CAMjD;AAED;;;;;GAKG;AACH,wBAAsB,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC,CAMpD;AAED;;;;;;;GAOG;AACH,iEAAiE;AACjE,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAGxD;AA0BD;;;;;;GAMG;AACH,wBAAgB,mBAAmB,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAM3G;AAED;;;;;;;;GAQG;AACH,wBAAsB,eAAe,IAAI,OAAO,CAAC,MAAM,CAAC,CAoBvD;AAED,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAM3C;AAED,wBAAsB,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAoFjF;AAED;;;;GAIG;AACH,MAAM,WAAW,eAAe;IAC9B,mEAAmE;IACnE,UAAU,CAAC,EAAE,aAAa,EAAE,CAAA;IAC5B,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,iBAAiB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IAC/B,WAAW,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACzB,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,kBAAkB,CAAC,EAAE,MAAM,CAAA;CAC5B;AAED,oDAAoD;AACpD,UAAU,WAAY,SAAQ,eAAe;IAC3C,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CACvB;AAED;;;;;;;;;GASG;AACH,wBAAsB,gBAAgB,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CA4B3E;AAED,wBAAsB,YAAY,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,SAAS,CAAC,CAkCxE;AAED,6EAA6E;AAC7E,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,MAAM,CAAA;IAChB,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CACvB;AAED,wEAAwE;AACxE,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAErD;AAED;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAAC,YAAY,EAAE,MAAM,GAAG,MAAM,CAE/D;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,cAAc,GAAG,SAAS,CAgD/D;AAED;;;;;;;;GAQG;AACH,wBAAsB,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CA4CvG;AAED;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CAAC,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,OAAO,CAE3F;AAED;;;;;;;;;GASG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,IAAI,CAmE9D;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAC5B,KAAK,EAAE,UAAU,EACjB,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EACxB,KAAK,GAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAc,GACrC,IAAI,CAoDN"}
|
package/dist/indexer.js
CHANGED
|
@@ -22,8 +22,10 @@ let cursorCheckpointTimer = null;
|
|
|
22
22
|
*
|
|
23
23
|
* The relay's `subscribeRepos` seq and Jetstream's seq are different
|
|
24
24
|
* coordinate systems — resuming one from the other's value would either skip
|
|
25
|
-
* a swathe of the stream or replay from a nonsense offset.
|
|
26
|
-
*
|
|
25
|
+
* a swathe of the stream or replay from a nonsense offset. So is one Jetstream
|
|
26
|
+
* instance's seq against another's, which is why the key names the instance
|
|
27
|
+
* rather than the kind. Each source owns its own key so switching between them
|
|
28
|
+
* (or back) is safe.
|
|
27
29
|
*/
|
|
28
30
|
let cursorKey = 'relay';
|
|
29
31
|
const BATCH_SIZE = 100;
|
|
@@ -517,6 +519,18 @@ export async function startIndexer(opts) {
|
|
|
517
519
|
export function auxCursorKey(relayUrl) {
|
|
518
520
|
return `relay:${relayUrl}`;
|
|
519
521
|
}
|
|
522
|
+
/**
|
|
523
|
+
* The `_cursor` row a Jetstream instance persists its position to.
|
|
524
|
+
*
|
|
525
|
+
* Named for the instance, because a Jetstream sequence means nothing on
|
|
526
|
+
* another one. Pointed at a second instance — a region swap, an outage
|
|
527
|
+
* worked around — a shared row would hand over a number from somewhere else,
|
|
528
|
+
* and the stream would resume at whatever that happens to address there:
|
|
529
|
+
* no error, no refusal, just a silent jump past whatever fell in between.
|
|
530
|
+
*/
|
|
531
|
+
export function jetstreamCursorKey(jetstreamUrl) {
|
|
532
|
+
return `jetstream:${jetstreamUrl}`;
|
|
533
|
+
}
|
|
520
534
|
/**
|
|
521
535
|
* Tail a second `subscribeRepos` alongside the primary stream.
|
|
522
536
|
*
|
package/dist/jetstream.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"jetstream.d.ts","sourceRoot":"","sources":["../src/jetstream.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,
|
|
1
|
+
{"version":3,"file":"jetstream.d.ts","sourceRoot":"","sources":["../src/jetstream.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAUL,KAAK,QAAQ,EACb,KAAK,eAAe,EACrB,MAAM,cAAc,CAAA;AAGrB,sEAAsE;AACtE,eAAO,MAAM,eAAe,MAAM,CAAA;AAClC,eAAO,MAAM,QAAQ,QAAS,CAAA;AAO9B;;;;;;;;;;;;;;;;GAgBG;AACH,eAAO,MAAM,kBAAkB,IAAI,CAAA;AAEnC,MAAM,WAAW,aAAc,SAAQ,eAAe;IACpD,sEAAsE;IACtE,YAAY,EAAE,MAAM,CAAA;IACpB,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CACvB;AAED,wDAAwD;AACxD,UAAU,eAAe;IACvB,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;CAC7B;AAED;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAC/B,YAAY,EAAE,MAAM,EACpB,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EACxB,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,EAC/B,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GACrB,MAAM,CASR;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,GAAG,IAAI,CAalG;AAED;;;;;;;;GAQG;AACH,wBAAgB,UAAU,CAAC,MAAM,EAAE,eAAe,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,QAAQ,GAAG,IAAI,CAY7F;AAED;;;;;;GAMG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,CAuBzE;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,MAAM,GAAG,IAAI,EACtB,UAAU,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GACpC,MAAM,GAAG,IAAI,GAAG,SAAS,CAK3B;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,qBAAqB,CAAC,IAAI,EAAE,aAAa,EAAE,QAAQ,SAAI,GAAG,OAAO,CAAC,SAAS,CAAC,CA0DjG"}
|
package/dist/jetstream.js
CHANGED
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
* @see https://bsky.network/docs/jetstream
|
|
15
15
|
* @module
|
|
16
16
|
*/
|
|
17
|
-
import { applyCommit, configureIndexer, getLastSeq, handleIdentityEvent, isIndexableCollection, noteSeq, resumeCursor, setCursorKey, } from "./indexer.js";
|
|
17
|
+
import { applyCommit, configureIndexer, getLastSeq, handleIdentityEvent, isIndexableCollection, jetstreamCursorKey, noteSeq, resumeCursor, setCursorKey, } from "./indexer.js";
|
|
18
18
|
import { log, emit } from "./logger.js";
|
|
19
19
|
/** Server-side filter caps, rejected before the WebSocket upgrade. */
|
|
20
20
|
export const MAX_COLLECTIONS = 100;
|
|
@@ -170,7 +170,7 @@ export async function startJetstreamIndexer(opts, refusals = 0) {
|
|
|
170
170
|
const { jetstreamUrl, collections } = opts;
|
|
171
171
|
const pinnedRepos = opts.pinnedRepos || null;
|
|
172
172
|
assertFilterLimits(collections, pinnedRepos);
|
|
173
|
-
setCursorKey(
|
|
173
|
+
setCursorKey(jetstreamCursorKey(opts.jetstreamUrl));
|
|
174
174
|
await configureIndexer(opts);
|
|
175
175
|
const cursor = reconnectCursor(refusals, getLastSeq(), opts.cursor);
|
|
176
176
|
const wsUrl = buildSubscribeUrl(jetstreamUrl, collections, pinnedRepos, cursor);
|
package/dist/main.js
CHANGED
|
@@ -22,7 +22,7 @@ import { initFeeds, listFeeds } from "./feeds.js";
|
|
|
22
22
|
import { initXrpc, listXrpc, configureRelay, configureCdn, configureOAuth, callXrpc } from "./xrpc.js";
|
|
23
23
|
import { initOpengraph } from "./opengraph.js";
|
|
24
24
|
import { initLabels, getLabelDefinitions } from "./labels.js";
|
|
25
|
-
import { auxCursorKey, startAuxIndexer, startIndexer, sweepReferences } from "./indexer.js";
|
|
25
|
+
import { auxCursorKey, jetstreamCursorKey, startAuxIndexer, startIndexer, sweepReferences } from "./indexer.js";
|
|
26
26
|
import { startJetstreamIndexer } from "./jetstream.js";
|
|
27
27
|
import { rebuildAllIndexes } from "./database/fts.js";
|
|
28
28
|
import { createHandler, registerCoreHandlers } from "./server.js";
|
|
@@ -237,7 +237,7 @@ const indexerCore = {
|
|
|
237
237
|
ftsRebuildInterval: config.ftsRebuildInterval,
|
|
238
238
|
};
|
|
239
239
|
if (config.jetstream) {
|
|
240
|
-
const cursor = ignoreSavedCursor ? null : await getCursor(
|
|
240
|
+
const cursor = ignoreSavedCursor ? null : await getCursor(jetstreamCursorKey(config.jetstream.url));
|
|
241
241
|
startJetstreamIndexer({ ...indexerCore, jetstreamUrl: config.jetstream.url, cursor });
|
|
242
242
|
}
|
|
243
243
|
else {
|