@mnmal-ai/substrate 1.0.0-alpha.1 → 1.0.0-alpha.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +48 -6
- package/dist/boot-store-BVqfFxh1.js +4060 -0
- package/dist/{runtime-metrics-DwpDJ8vz.d.ts → bridge-Cp4PaIYO.d.ts} +2 -87
- package/dist/cli-hSCBywhE.js +3126 -0
- package/dist/client.d.ts +130 -8
- package/dist/client.js +6 -5
- package/dist/{plugin-t2g2wX0b.d.ts → dataset-CejF-Fr_.d.ts} +1640 -1320
- package/dist/{agent-token-BIboWOU0.js → env-utils-CGNazUNt.js} +31 -2
- package/dist/{errors-D4vKuwz8.d.ts → errors-7mycvNGq.d.ts} +1 -1
- package/dist/{index-D7mSOFlM.d.ts → index-DgSfwx7M.d.ts} +1 -1
- package/dist/node-version-C5FSmwJX.js +26 -0
- package/dist/pack.d.ts +2 -2
- package/dist/pack.js +1 -1
- package/dist/pg-edges-store-Lxsl5ZnK.js +381 -0
- package/dist/{plugin-BuQQEzL-.js → plugin-CtwOINCM.js} +171 -18
- package/dist/proposals-bootstrap.js +6 -4
- package/dist/proposals.config.schema.json +5 -0
- package/dist/proposals.d.ts +15 -4
- package/dist/proposals.default-config.json +2 -1
- package/dist/proposals.js +2 -2
- package/dist/recall-bootstrap.js +13 -4
- package/dist/recall.config.schema.json +12 -0
- package/dist/recall.d.ts +54 -9
- package/dist/recall.js +2 -2
- package/dist/{repl-C52wyfMs.js → repl-DqfHelK7.js} +62 -29
- package/dist/{row-ByLFE2Wx.d.ts → row-BKKCXIG6.d.ts} +5 -3
- package/dist/run-B5dTbZ3g.js +1997 -0
- package/dist/{schema-g4FQLr0V.d.ts → schema-DoSc-4ig.d.ts} +44 -7
- package/dist/script-C6DSa_Na.js +134 -0
- package/dist/sdk.d.ts +90 -12
- package/dist/sdk.js +5 -110
- package/dist/{src-YXlv_RyL.js → src-5SJ5GEIM.js} +1555 -326
- package/dist/{src-DuopV8u7.js → src-7CUSMK66.js} +148 -19
- package/dist/{src-zcyUofMJ.js → src-Bi849yhn.js} +39 -15
- package/dist/{src-5hVbeiAu.js → src-CqT-QWZ_.js} +697 -57
- package/dist/src-CyvV7dCK.js +4333 -0
- package/dist/{row-DK7lPZVe.js → src-Da0t6si5.js} +11 -11
- package/dist/{src-C4DzaIQZ.js → src-hZb8wfvg.js} +12 -1
- package/dist/substrate-admin.js +17 -781
- package/dist/substrate.js +4 -5476
- package/dist/testing.d.ts +557 -8
- package/dist/testing.js +39 -3
- package/dist/{ws-C72iVS5T.d.ts → ws-ByZyHk7m.d.ts} +3 -3
- package/package.json +21 -20
- package/dist/dataset-DOXGRz50.js +0 -1535
- package/dist/env-utils-B7SGmoeJ.js +0 -21
- package/dist/src-CjEwiKXw.js +0 -2327
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
2
|
+
import { c as makeTokenProvider, i as DEFAULT_CAPABILITIES, n as trimTrailingSlashes } from "./env-utils-CGNazUNt.js";
|
|
3
|
+
import { D as isActiveAdminKeyRow, O as capabilitiesFor, T as SYSTEM_NAMESPACE } from "./src-Da0t6si5.js";
|
|
4
|
+
import { C as parseQualifiedKey, _ as decodeEnvelope, a as buildDecodePlan, g as SchemaDriftError, i as applyDecodePlanInPlace, o as decodeWsEnvelope, s as encodeWsFrame, t as WS_SUBPROTOCOL_SUBSTRATE_PACK } from "./src-hZb8wfvg.js";
|
|
4
5
|
import { signal } from "@preact/signals-core";
|
|
5
6
|
import { WebSocket } from "ws";
|
|
6
7
|
|
|
@@ -26,6 +27,7 @@ const CHECK_ADMIN = "admin-capability";
|
|
|
26
27
|
const CHECK_FLOOR = "admin-floor";
|
|
27
28
|
const CHECK_NAMESPACES = "namespaces";
|
|
28
29
|
const CHECK_DIR = "key-directory";
|
|
30
|
+
const CHECK_STORAGE = "storage";
|
|
29
31
|
/** Every check downstream of a failure, so a skip is never silent. */
|
|
30
32
|
const DOWNSTREAM = {
|
|
31
33
|
[CHECK_REACHABLE]: [
|
|
@@ -35,21 +37,28 @@ const DOWNSTREAM = {
|
|
|
35
37
|
CHECK_ADMIN,
|
|
36
38
|
CHECK_FLOOR,
|
|
37
39
|
CHECK_NAMESPACES,
|
|
38
|
-
CHECK_DIR
|
|
40
|
+
CHECK_DIR,
|
|
41
|
+
CHECK_STORAGE
|
|
39
42
|
],
|
|
40
43
|
[CHECK_CREDENTIAL]: [
|
|
41
44
|
CHECK_IDENTITY,
|
|
42
45
|
CHECK_ADMIN,
|
|
43
46
|
CHECK_FLOOR,
|
|
44
47
|
CHECK_NAMESPACES,
|
|
45
|
-
CHECK_DIR
|
|
48
|
+
CHECK_DIR,
|
|
49
|
+
CHECK_STORAGE
|
|
46
50
|
],
|
|
47
51
|
[CHECK_IDENTITY]: [
|
|
48
52
|
CHECK_ADMIN,
|
|
49
53
|
CHECK_FLOOR,
|
|
50
|
-
CHECK_DIR
|
|
54
|
+
CHECK_DIR,
|
|
55
|
+
CHECK_STORAGE
|
|
56
|
+
],
|
|
57
|
+
[CHECK_ADMIN]: [
|
|
58
|
+
CHECK_FLOOR,
|
|
59
|
+
CHECK_DIR,
|
|
60
|
+
CHECK_STORAGE
|
|
51
61
|
],
|
|
52
|
-
[CHECK_ADMIN]: [CHECK_FLOOR, CHECK_DIR],
|
|
53
62
|
[CHECK_FLOOR]: [CHECK_DIR]
|
|
54
63
|
};
|
|
55
64
|
var Checks = class {
|
|
@@ -275,7 +284,8 @@ const DIAGNOSE_CHECKS = [
|
|
|
275
284
|
CHECK_ADMIN,
|
|
276
285
|
CHECK_FLOOR,
|
|
277
286
|
CHECK_NAMESPACES,
|
|
278
|
-
CHECK_DIR
|
|
287
|
+
CHECK_DIR,
|
|
288
|
+
CHECK_STORAGE
|
|
279
289
|
];
|
|
280
290
|
/**
|
|
281
291
|
* Runs every check against `opts.baseUrl`, in dependency order. A failure
|
|
@@ -285,7 +295,7 @@ async function diagnose(opts) {
|
|
|
285
295
|
const checks = new Checks();
|
|
286
296
|
const fetchImpl = opts.fetch ?? fetch;
|
|
287
297
|
const adminCapability = opts.adminCapability ?? "admin";
|
|
288
|
-
const baseUrl = opts.baseUrl
|
|
298
|
+
const baseUrl = trimTrailingSlashes(opts.baseUrl);
|
|
289
299
|
let identityId = null;
|
|
290
300
|
let facts = {};
|
|
291
301
|
const done = () => ({
|
|
@@ -349,7 +359,8 @@ async function diagnose(opts) {
|
|
|
349
359
|
for (const name of [
|
|
350
360
|
CHECK_ADMIN,
|
|
351
361
|
CHECK_FLOOR,
|
|
352
|
-
CHECK_DIR
|
|
362
|
+
CHECK_DIR,
|
|
363
|
+
CHECK_STORAGE
|
|
353
364
|
]) checks.skip(name, "not asked — this identity was not expected to be an admin");
|
|
354
365
|
return done();
|
|
355
366
|
}
|
|
@@ -363,6 +374,16 @@ async function diagnose(opts) {
|
|
|
363
374
|
return done();
|
|
364
375
|
}
|
|
365
376
|
checks.ok(CHECK_ADMIN, `the grant confers '${adminCapability}' on ${SYSTEM_NAMESPACE} — this identity can administer`);
|
|
377
|
+
const config = await getJson(`${baseUrl}/substrate/v1/config`, fetchImpl, { authorization });
|
|
378
|
+
const storage = "error" in config ? void 0 : config.body.storage;
|
|
379
|
+
if ("error" in config || config.status !== 200) checks.skip(CHECK_STORAGE, `not asked — /substrate/v1/config ${"error" in config ? `could not be reached: ${config.error}` : `answered HTTP ${config.status}`}`);
|
|
380
|
+
else if (storage === void 0) checks.skip(CHECK_STORAGE, "not asked — the server reports no storage section (memory store)");
|
|
381
|
+
else {
|
|
382
|
+
const warnings = Array.isArray(storage.warnings) ? storage.warnings.map(String) : [];
|
|
383
|
+
const info = Array.isArray(storage.info) ? storage.info.map((l) => `\n${String(l)}`).join("") : "";
|
|
384
|
+
if (warnings.length === 0) checks.ok(CHECK_STORAGE, `${String(storage.layout)} layout, no warnings${info}`);
|
|
385
|
+
else checks.no(CHECK_STORAGE, `${String(storage.layout)} layout, ${warnings.length} warning(s):\n${warnings.join("\n")}${info}`, "Each warning names what to move or drop; indexes are never dropped automatically.");
|
|
386
|
+
}
|
|
366
387
|
const floor = await listLiveKeys(baseUrl, authorization, fetchImpl);
|
|
367
388
|
if ("error" in floor) {
|
|
368
389
|
checks.no(CHECK_FLOOR, `could not read ${SYSTEM_NAMESPACE}/Key: ${floor.error}`, "The identity is admin, so this is a route or store problem rather than a permission one — read `journalctl -u substrate-binary`.");
|
|
@@ -886,6 +907,12 @@ function reduceSubscribeRows(current, qualifiedKey, event) {
|
|
|
886
907
|
default: return current;
|
|
887
908
|
}
|
|
888
909
|
}
|
|
910
|
+
/** End reasons after which the transport resubscribes rather than finishing. */
|
|
911
|
+
const SURVIVABLE_END_REASONS = /* @__PURE__ */ new Set([
|
|
912
|
+
"server_restart",
|
|
913
|
+
"credential_expired",
|
|
914
|
+
"credential_changed"
|
|
915
|
+
]);
|
|
889
916
|
/**
|
|
890
917
|
* Fold a single-key subscription's event stream into a live row-set: yields the
|
|
891
918
|
* current `T[]` after every snapshot / add / update / remove. Completes on `end`
|
|
@@ -901,7 +928,8 @@ function reduceSubscribeRows(current, qualifiedKey, event) {
|
|
|
901
928
|
async function* watchRows(events, qualifiedKey) {
|
|
902
929
|
let rows = [];
|
|
903
930
|
for await (const event of events) {
|
|
904
|
-
if (event.kind === "end") return;
|
|
931
|
+
if (event.kind === "end" && !SURVIVABLE_END_REASONS.has(event.reason ?? "")) return;
|
|
932
|
+
if (event.kind === "end") continue;
|
|
905
933
|
if (event.kind === "error") {
|
|
906
934
|
const first = event.errors[0];
|
|
907
935
|
throw new SubscribeError(first?.message ?? "subscribe error", first?.code ?? "unknown");
|
|
@@ -1108,7 +1136,7 @@ function createDefaultDiagnosticsSink() {
|
|
|
1108
1136
|
seen.add(dedupeKey);
|
|
1109
1137
|
const parts = [];
|
|
1110
1138
|
if (fields.length > 0) parts.push(`projected field(s) the type does not declare: ${fields.join(", ")}. The read succeeded — these are null on EVERY row, which is indistinguishable from a declared field that is always null. Check for a typo, or a substrate that has not restarted with your schema yet.`);
|
|
1111
|
-
if (filterValidationSkipped !== void 0) parts.push(`the where-clause was not schema-validated (${filterValidationSkipped}), so
|
|
1139
|
+
if (filterValidationSkipped !== void 0) parts.push(`the where-clause was not schema-validated (${filterValidationSkipped}), so its operators and operands were never checked against the field kinds. Results are still correct — the substrate refuses a predicate it cannot honour — but a malformed filter fails later and less clearly.`);
|
|
1112
1140
|
if (unrecognized.length > 0) parts.push(`the substrate reported diagnostic(s) this client cannot read: ${unrecognized.join(", ")}. Upgrade @mnmal-ai/substrate-client to surface them.`);
|
|
1113
1141
|
console.warn(`[substrate-client] ${opKey}: ${parts.join(" Also, ")} (Pass \`onDiagnostics\` to SubstrateClient to route this elsewhere.)`);
|
|
1114
1142
|
};
|
|
@@ -1271,7 +1299,7 @@ var HttpTransport = class {
|
|
|
1271
1299
|
#getToken;
|
|
1272
1300
|
constructor(opts) {
|
|
1273
1301
|
if (opts.token !== void 0 && opts.getToken !== void 0) throw new Error("HttpTransport: provide `token` OR `getToken`, not both");
|
|
1274
|
-
this.#endpoint = opts.endpoint
|
|
1302
|
+
this.#endpoint = trimTrailingSlashes(opts.endpoint);
|
|
1275
1303
|
this.#fetch = (opts.fetch ?? globalThis.fetch).bind(globalThis);
|
|
1276
1304
|
this.#token = opts.token;
|
|
1277
1305
|
this.#getToken = opts.getToken;
|
|
@@ -1643,6 +1671,13 @@ var WsTransport = class {
|
|
|
1643
1671
|
*/
|
|
1644
1672
|
#lastCloseCode = null;
|
|
1645
1673
|
/**
|
|
1674
|
+
* Backoff for closes that come AFTER an accepted upgrade (1013 overload,
|
|
1675
|
+
* 4001 on a lapsed credential). It lives on the transport, not the open
|
|
1676
|
+
* loop, so repeated churn grows the delay; a socket that delivers a frame
|
|
1677
|
+
* and outlives `maxDelayMs` proves healthy and resets it.
|
|
1678
|
+
*/
|
|
1679
|
+
#churn;
|
|
1680
|
+
/**
|
|
1646
1681
|
* Retry forever only once a socket has opened. Before that, a failure is
|
|
1647
1682
|
* most likely a wrong endpoint or a broken token, so it surfaces instead.
|
|
1648
1683
|
*/
|
|
@@ -1657,6 +1692,7 @@ var WsTransport = class {
|
|
|
1657
1692
|
...DEFAULT_RECONNECT,
|
|
1658
1693
|
...opts.reconnect
|
|
1659
1694
|
};
|
|
1695
|
+
this.#churn = new ReconnectScheduler(this.#reconnectOpts);
|
|
1660
1696
|
this.#encoding = opts.encoding ?? "json";
|
|
1661
1697
|
this.#overloadResubscribe = {
|
|
1662
1698
|
...DEFAULT_OVERLOAD_RESUBSCRIBE,
|
|
@@ -1715,12 +1751,21 @@ var WsTransport = class {
|
|
|
1715
1751
|
async #openLoop() {
|
|
1716
1752
|
const scheduler = new ReconnectScheduler(this.#reconnectOpts);
|
|
1717
1753
|
const wasGracefulRestart = this.#lastCloseCode === 1012;
|
|
1754
|
+
const churned = this.#lastCloseCode === 1013 || this.#lastCloseCode === 4001;
|
|
1718
1755
|
this.#lastCloseCode = null;
|
|
1756
|
+
if (churned) {
|
|
1757
|
+
await this.#churn.schedule();
|
|
1758
|
+
if (this.#destroyed) throw new NetworkError("WsTransport destroyed", null);
|
|
1759
|
+
}
|
|
1719
1760
|
let firstFailure = true;
|
|
1720
1761
|
while (true) {
|
|
1721
1762
|
if (this.#destroyed) throw new NetworkError("WsTransport destroyed", null);
|
|
1722
1763
|
try {
|
|
1723
1764
|
const mux = await this.#openOnce();
|
|
1765
|
+
if (this.#destroyed) {
|
|
1766
|
+
mux.close();
|
|
1767
|
+
throw new NetworkError("WsTransport destroyed", null);
|
|
1768
|
+
}
|
|
1724
1769
|
scheduler.reset();
|
|
1725
1770
|
this.#hasOpened = true;
|
|
1726
1771
|
this.#mux = mux;
|
|
@@ -1776,9 +1821,16 @@ var WsTransport = class {
|
|
|
1776
1821
|
rawWs.addEventListener("error", onError);
|
|
1777
1822
|
});
|
|
1778
1823
|
const mux = new MultiplexedSocket(ws, { usePack: this.#encoding === "pack" });
|
|
1824
|
+
const openedAt = Date.now();
|
|
1825
|
+
let delivered = false;
|
|
1826
|
+
ws.addEventListener("message", () => {
|
|
1827
|
+
delivered = true;
|
|
1828
|
+
}, { once: true });
|
|
1779
1829
|
ws.addEventListener("close", (ev) => {
|
|
1780
1830
|
if (this.#destroyed) return;
|
|
1781
|
-
|
|
1831
|
+
const healthy = delivered && Date.now() - openedAt >= this.#reconnectOpts.maxDelayMs;
|
|
1832
|
+
if (healthy) this.#churn.reset();
|
|
1833
|
+
if (ev.code === 4001 && !(healthy && this.#canRefreshToken())) {
|
|
1782
1834
|
if (this.#mux === mux) this.#mux = null;
|
|
1783
1835
|
this.#failActiveSubs(new NetworkError(`WS closed 4001 unauthorized by ${this.#endpoint} — bearer rejected (wrong/expired token, or none configured against an auth-gated server)`, ev));
|
|
1784
1836
|
return;
|
|
@@ -1791,6 +1843,10 @@ var WsTransport = class {
|
|
|
1791
1843
|
});
|
|
1792
1844
|
return mux;
|
|
1793
1845
|
}
|
|
1846
|
+
/** Only a provider can mint a fresh token; a fixed `token` would be refused again. */
|
|
1847
|
+
#canRefreshToken() {
|
|
1848
|
+
return this.#token === void 0 && this.#getToken !== void 0;
|
|
1849
|
+
}
|
|
1794
1850
|
/** Terminate every active sub with `err` — delivered so next() throws. */
|
|
1795
1851
|
#failActiveSubs(err) {
|
|
1796
1852
|
for (const [, sub] of this.#activeSubs) {
|
|
@@ -1925,6 +1981,7 @@ var WsTransport = class {
|
|
|
1925
1981
|
if (msg.kind === "end") {
|
|
1926
1982
|
if (msg.reason === REASON_SUBSCRIPTION_OVERLOADED && cfg.enabled && sub.retryTimer !== null) return "suppress";
|
|
1927
1983
|
if (msg.reason === REASON_SERVER_RESTART) return "survive";
|
|
1984
|
+
if ((msg.reason === "credential_expired" || msg.reason === "credential_changed") && this.#canRefreshToken()) return "survive";
|
|
1928
1985
|
return "terminal";
|
|
1929
1986
|
}
|
|
1930
1987
|
return "data";
|
|
@@ -2129,7 +2186,7 @@ var SubstrateClient = class {
|
|
|
2129
2186
|
...opts.getToken !== void 0 ? { getToken: opts.getToken } : {},
|
|
2130
2187
|
...opts.fetch !== void 0 ? { fetch: opts.fetch } : {}
|
|
2131
2188
|
});
|
|
2132
|
-
this.#wsEndpoint = `${opts.endpoint.replace(/^http/, "ws")
|
|
2189
|
+
this.#wsEndpoint = `${trimTrailingSlashes(opts.endpoint.replace(/^http/, "ws"))}/substrate/v1/ws`;
|
|
2133
2190
|
this.#wsToken = opts.token;
|
|
2134
2191
|
this.#wsGetToken = opts.getToken;
|
|
2135
2192
|
this.#wsEncoding = opts.wsEncoding ?? "json";
|
|
@@ -2272,10 +2329,19 @@ var SubstrateClient = class {
|
|
|
2272
2329
|
async #decode(raw, frame, wire) {
|
|
2273
2330
|
let env;
|
|
2274
2331
|
if (wire === "pack" && isPackMarker(raw)) {
|
|
2275
|
-
const
|
|
2276
|
-
const
|
|
2277
|
-
|
|
2278
|
-
|
|
2332
|
+
const bytes = raw.__packBytes;
|
|
2333
|
+
const decode = (expectedSchemaVersion) => decodeEnvelope(bytes, {
|
|
2334
|
+
...this.#buildDecodeCtx(frame, this.#schemaCache ?? {}),
|
|
2335
|
+
...expectedSchemaVersion !== void 0 ? { expectedSchemaVersion } : {}
|
|
2336
|
+
});
|
|
2337
|
+
try {
|
|
2338
|
+
env = decode(this.#driftCachedVersion ?? void 0);
|
|
2339
|
+
} catch (err) {
|
|
2340
|
+
if (!(err instanceof SchemaDriftError)) throw err;
|
|
2341
|
+
await this.#refetchSchema();
|
|
2342
|
+
this.#driftCachedVersion = err.envelopeSchemaVersion;
|
|
2343
|
+
env = decode();
|
|
2344
|
+
}
|
|
2279
2345
|
} else env = decodeJsonEnvelope(raw);
|
|
2280
2346
|
if (this.#driftCachedVersion === null) this.#driftCachedVersion = env.schemaVersion;
|
|
2281
2347
|
else if (this.#driftCachedVersion !== env.schemaVersion) {
|
|
@@ -2405,4 +2471,67 @@ function noRedirectFetch(fetchImpl = fetch) {
|
|
|
2405
2471
|
}
|
|
2406
2472
|
|
|
2407
2473
|
//#endregion
|
|
2408
|
-
|
|
2474
|
+
//#region ../../packages/substrate-client/src/from-env.ts
|
|
2475
|
+
/**
|
|
2476
|
+
* The capabilities a client asks for, from `SUBSTRATE_AGENT_CAPABILITIES` (csv),
|
|
2477
|
+
* defaulting to `substrate-policy`'s default.
|
|
2478
|
+
*
|
|
2479
|
+
* Not `ADMIN_CAPABILITIES`: a dataset-registry key intersects the claim, but a
|
|
2480
|
+
* file-registry key takes it verbatim, so a wider default would promote every
|
|
2481
|
+
* agent on such a deployment to admin.
|
|
2482
|
+
*/
|
|
2483
|
+
function resolveAgentCapabilities(env) {
|
|
2484
|
+
const raw = env.SUBSTRATE_AGENT_CAPABILITIES;
|
|
2485
|
+
if (raw === void 0 || raw.trim().length === 0) return DEFAULT_CAPABILITIES;
|
|
2486
|
+
const parsed = raw.split(",").map((c) => c.trim()).filter((c) => c.length > 0);
|
|
2487
|
+
return parsed.length > 0 ? parsed : DEFAULT_CAPABILITIES;
|
|
2488
|
+
}
|
|
2489
|
+
/**
|
|
2490
|
+
* Builds a client from `SUBSTRATE_URL` and `SUBSTRATE_AGENT_KEY_ID` +
|
|
2491
|
+
* `SUBSTRATE_AGENT_KEY_PATH`, signing every request as that kid.
|
|
2492
|
+
*
|
|
2493
|
+
* Redirects are refused and the token's audience is the endpoint's origin.
|
|
2494
|
+
*
|
|
2495
|
+
* @throws Error when `SUBSTRATE_URL` is unset or unsafe for a credential, when
|
|
2496
|
+
* only one of the key pair is set, when the key cannot be read, or when no
|
|
2497
|
+
* key is set and `allowAnonymous` is not.
|
|
2498
|
+
*/
|
|
2499
|
+
function createClientFromEnv(opts = {}) {
|
|
2500
|
+
const { env = process.env, allowAnonymous = false, fetch: fetchImpl, ...rest } = opts;
|
|
2501
|
+
const endpoint = trimTrailingSlashes(env.SUBSTRATE_URL ?? "");
|
|
2502
|
+
if (endpoint === "") throw new Error("SUBSTRATE_URL is required");
|
|
2503
|
+
const keyPath = env.SUBSTRATE_AGENT_KEY_PATH ?? "";
|
|
2504
|
+
const kid = env.SUBSTRATE_AGENT_KEY_ID ?? "";
|
|
2505
|
+
if (keyPath === "" !== (kid === "")) throw new Error(`SUBSTRATE_AGENT_KEY_PATH and SUBSTRATE_AGENT_KEY_ID must be set together; only ${keyPath !== "" ? "SUBSTRATE_AGENT_KEY_PATH" : "SUBSTRATE_AGENT_KEY_ID"} is set`);
|
|
2506
|
+
const base = {
|
|
2507
|
+
...rest,
|
|
2508
|
+
endpoint,
|
|
2509
|
+
fetch: noRedirectFetch(fetchImpl)
|
|
2510
|
+
};
|
|
2511
|
+
if (kid === "") {
|
|
2512
|
+
if (!allowAnonymous) throw new Error("no signing key: set SUBSTRATE_AGENT_KEY_ID and SUBSTRATE_AGENT_KEY_PATH, or pass allowAnonymous");
|
|
2513
|
+
return {
|
|
2514
|
+
client: new SubstrateClient(base),
|
|
2515
|
+
endpoint,
|
|
2516
|
+
kid: null
|
|
2517
|
+
};
|
|
2518
|
+
}
|
|
2519
|
+
const getToken = makeTokenProvider({
|
|
2520
|
+
keyPath,
|
|
2521
|
+
kid
|
|
2522
|
+
}, {
|
|
2523
|
+
capabilities: resolveAgentCapabilities(env),
|
|
2524
|
+
audience: tokenAudience(endpoint)
|
|
2525
|
+
});
|
|
2526
|
+
return {
|
|
2527
|
+
client: new SubstrateClient({
|
|
2528
|
+
...base,
|
|
2529
|
+
getToken
|
|
2530
|
+
}),
|
|
2531
|
+
endpoint,
|
|
2532
|
+
kid
|
|
2533
|
+
};
|
|
2534
|
+
}
|
|
2535
|
+
|
|
2536
|
+
//#endregion
|
|
2537
|
+
export { diagnose as C, DIAGNOSE_CHECKS as S, SKIPPABLE_STEPS as _, SubstrateClient as a, planOnboarding as b, reduceSubscribeRows as c, NetworkError as d, SubscribeError as f, ONBOARD_STEPS as g, CURRENT_MARK as h, tokenAudience as i, watchRows as l, BACKFILL_CHUNK as m, resolveAgentCapabilities as n, collectDiagnostics as o, AdminBackfillError as p, noRedirectFetch as r, createDefaultDiagnosticsSink as s, createClientFromEnv as t, AuthError as u, STEP_MARKS as v, reconcileIdentities as w, requiredOutstanding as x, currentStep as y };
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
2
|
+
import { n as txScopeIdFor } from "./src-CyvV7dCK.js";
|
|
3
|
+
import { B as ROW_FILTER_FIELD, V as matchesRowFilter, W as inFlightError, Y as unresolvedRow, Z as cosineSimilarity, et as memberKey } from "./plugin-CtwOINCM.js";
|
|
4
4
|
|
|
5
5
|
//#region ../../plugins/substrate-recall/package.json
|
|
6
|
-
var version = "1.0.0-alpha.
|
|
6
|
+
var version = "1.0.0-alpha.10";
|
|
7
7
|
|
|
8
8
|
//#endregion
|
|
9
9
|
//#region ../../plugins/substrate-recall/src/in-memory-cluster-store.ts
|
|
@@ -68,7 +68,8 @@ var InMemoryClusterStore = class {
|
|
|
68
68
|
return found === void 0 ? null : this.#copyCluster(found);
|
|
69
69
|
}
|
|
70
70
|
async listMembers(clusterId, opts = {}) {
|
|
71
|
-
|
|
71
|
+
const ids = new Set(typeof clusterId === "string" ? [clusterId] : clusterId);
|
|
72
|
+
return this.#members.filter((m) => ids.has(m.clusterId)).filter((m) => opts.includeDeparted === true || m.departedAt === null).map((m) => ({ ...m }));
|
|
72
73
|
}
|
|
73
74
|
async listAllMembers() {
|
|
74
75
|
return this.#members.filter((m) => m.departedAt === null).map((m) => ({ ...m }));
|
|
@@ -195,6 +196,8 @@ var InMemoryEmbeddingsStore = class {
|
|
|
195
196
|
async knnSearch(input) {
|
|
196
197
|
const threshold = typeof input.threshold === "number" ? input.threshold : 0;
|
|
197
198
|
const candidates = [];
|
|
199
|
+
const now = Date.now();
|
|
200
|
+
const halfLife = input.salienceHalfLifeHours;
|
|
198
201
|
for (const entry of this.#entries.values()) {
|
|
199
202
|
if (entry.rowNamespace !== input.namespace) continue;
|
|
200
203
|
if (entry.rowType !== input.type) continue;
|
|
@@ -218,7 +221,7 @@ var InMemoryEmbeddingsStore = class {
|
|
|
218
221
|
results.push({
|
|
219
222
|
row,
|
|
220
223
|
score,
|
|
221
|
-
salience: entry.salience,
|
|
224
|
+
salience: halfLife !== void 0 && halfLife > 0 ? decayed(entry.salience, now - entry.salienceUpdatedAt.getTime(), halfLife) : entry.salience,
|
|
222
225
|
...input.returnVector === true ? { vector: Array.from(entry.vector) } : {}
|
|
223
226
|
});
|
|
224
227
|
}
|
|
@@ -227,14 +230,12 @@ var InMemoryEmbeddingsStore = class {
|
|
|
227
230
|
async boostSalience(targets, halfLifeHours, bump = 1) {
|
|
228
231
|
if (targets.length === 0) return [];
|
|
229
232
|
const now = /* @__PURE__ */ new Date();
|
|
230
|
-
const halfLifeMs = halfLifeHours * 3600 * 1e3;
|
|
231
233
|
const applied = [];
|
|
232
234
|
for (const t of targets) {
|
|
233
235
|
const entry = this.#entries.get(key(t.namespace, t.type, t.id));
|
|
234
236
|
if (entry === void 0) continue;
|
|
235
|
-
const elapsedMs =
|
|
236
|
-
|
|
237
|
-
entry.salience = entry.salience * decay + bump;
|
|
237
|
+
const elapsedMs = now.getTime() - entry.salienceUpdatedAt.getTime();
|
|
238
|
+
entry.salience = decayed(entry.salience, elapsedMs, halfLifeHours) + bump;
|
|
238
239
|
entry.salienceUpdatedAt = now;
|
|
239
240
|
applied.push({
|
|
240
241
|
namespace: t.namespace,
|
|
@@ -297,6 +298,10 @@ var InMemoryEmbeddingsStore = class {
|
|
|
297
298
|
return out;
|
|
298
299
|
}
|
|
299
300
|
};
|
|
301
|
+
/** `salience` after `elapsedMs` of exponential decay at `halfLifeHours`. */
|
|
302
|
+
function decayed(salience, elapsedMs, halfLifeHours) {
|
|
303
|
+
return salience * Math.exp(-Math.LN2 * Math.max(0, elapsedMs) / (halfLifeHours * 3600 * 1e3));
|
|
304
|
+
}
|
|
300
305
|
|
|
301
306
|
//#endregion
|
|
302
307
|
//#region ../../plugins/substrate-recall/src/in-memory-lexical-store.ts
|
|
@@ -476,7 +481,7 @@ async function recountClusters(tx, clusterIds, stamps) {
|
|
|
476
481
|
WHERE c.id = counts.cluster_id
|
|
477
482
|
`;
|
|
478
483
|
}
|
|
479
|
-
/** Distinct from the migrator's `substrate-migrations` key;
|
|
484
|
+
/** Distinct from the migrator's `substrate-migrations` key; scoped by schema the same way. */
|
|
480
485
|
const FORMATION_LOCK_KEY = "substrate-recall-crystallise";
|
|
481
486
|
var PgClusterStore = class {
|
|
482
487
|
#sql;
|
|
@@ -487,7 +492,7 @@ var PgClusterStore = class {
|
|
|
487
492
|
}
|
|
488
493
|
/**
|
|
489
494
|
* Runs `fn` holding a session advisory lock that excludes formation in every
|
|
490
|
-
* process on this
|
|
495
|
+
* process on this schema; throws `crystallise_in_flight` if it is held.
|
|
491
496
|
*
|
|
492
497
|
* Lock and unlock share one reserved connection, since through the pool the
|
|
493
498
|
* unlock can land on another session and fail to release.
|
|
@@ -496,13 +501,13 @@ var PgClusterStore = class {
|
|
|
496
501
|
const conn = await this.#sql.reserve();
|
|
497
502
|
try {
|
|
498
503
|
const [row] = await conn`
|
|
499
|
-
SELECT pg_try_advisory_lock(hashtext(${FORMATION_LOCK_KEY})) AS acquired
|
|
504
|
+
SELECT pg_try_advisory_lock(hashtext(${FORMATION_LOCK_KEY} || ':' || current_schema())) AS acquired
|
|
500
505
|
`;
|
|
501
506
|
if (row?.acquired !== true) throw inFlightError();
|
|
502
507
|
try {
|
|
503
508
|
return await fn();
|
|
504
509
|
} finally {
|
|
505
|
-
await conn`SELECT pg_advisory_unlock(hashtext(${FORMATION_LOCK_KEY}))`;
|
|
510
|
+
await conn`SELECT pg_advisory_unlock(hashtext(${FORMATION_LOCK_KEY} || ':' || current_schema()))`;
|
|
506
511
|
}
|
|
507
512
|
} finally {
|
|
508
513
|
conn.release();
|
|
@@ -556,7 +561,7 @@ var PgClusterStore = class {
|
|
|
556
561
|
const sql = this.#sql;
|
|
557
562
|
return (await sql`
|
|
558
563
|
SELECT * FROM recall_cluster_members
|
|
559
|
-
WHERE cluster_id = ${clusterId}
|
|
564
|
+
WHERE cluster_id = ANY(${typeof clusterId === "string" ? [clusterId] : [...clusterId]}::text[])
|
|
560
565
|
${opts.includeDeparted === true ? sql`` : sql`AND departed_at IS NULL`}
|
|
561
566
|
ORDER BY short_id ASC
|
|
562
567
|
`).map(toMember);
|
|
@@ -729,6 +734,7 @@ var PgEmbeddingsStore = class {
|
|
|
729
734
|
const k = clampK(input.k);
|
|
730
735
|
const threshold = typeof input.threshold === "number" ? input.threshold : 0;
|
|
731
736
|
const wantVector = input.returnVector === true;
|
|
737
|
+
const halfLife = input.salienceHalfLifeHours;
|
|
732
738
|
return (await sql.begin(async (tx) => {
|
|
733
739
|
await tx`SET LOCAL hnsw.iterative_scan = 'strict_order'`;
|
|
734
740
|
await tx.unsafe(`SET LOCAL hnsw.ef_search = ${DEFAULT_HNSW_EF_SEARCH}`);
|
|
@@ -736,7 +742,7 @@ var PgEmbeddingsStore = class {
|
|
|
736
742
|
SELECT
|
|
737
743
|
${tx.unsafe(ROW_BODY_SQL)} AS row,
|
|
738
744
|
1 - (e.embedding <=> ${literal}::vector) AS score,
|
|
739
|
-
e.salience
|
|
745
|
+
e.salience ${halfLife !== void 0 && halfLife > 0 ? tx`* exp(-ln(2) * GREATEST(0, EXTRACT(EPOCH FROM (now() - e.salience_updated_at))) / ${halfLife * 3600})` : tx``} AS salience
|
|
740
746
|
${wantVector ? tx`, e.embedding::text AS embedding` : tx``}
|
|
741
747
|
FROM embeddings e
|
|
742
748
|
JOIN rows r
|
|
@@ -864,6 +870,24 @@ var PgEmbeddingsStore = class {
|
|
|
864
870
|
*
|
|
865
871
|
* @returns number of orphaned embeddings deleted.
|
|
866
872
|
*/
|
|
873
|
+
async staleRowIds(input) {
|
|
874
|
+
return (await this.#sql`
|
|
875
|
+
SELECT r.id FROM rows r
|
|
876
|
+
WHERE r.namespace = ${input.namespace}
|
|
877
|
+
AND r.type = ${input.type}
|
|
878
|
+
AND r.deleted_at IS NULL
|
|
879
|
+
AND r.id > ${input.afterId}
|
|
880
|
+
AND NOT EXISTS (
|
|
881
|
+
SELECT 1 FROM embeddings e
|
|
882
|
+
WHERE e.row_namespace = r.namespace
|
|
883
|
+
AND e.row_type = r.type
|
|
884
|
+
AND e.row_id = r.id
|
|
885
|
+
AND (r.updated_at IS NULL OR e.updated_at >= r.updated_at)
|
|
886
|
+
)
|
|
887
|
+
ORDER BY r.id
|
|
888
|
+
LIMIT ${input.limit}
|
|
889
|
+
`).map((r) => r.id);
|
|
890
|
+
}
|
|
867
891
|
async purgeOrphans() {
|
|
868
892
|
return (await this.#sql`
|
|
869
893
|
DELETE FROM embeddings e
|