@mnmal-ai/substrate 1.0.0-alpha.8 → 1.0.0-alpha.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/client.d.ts +1 -1
- package/dist/{dataset-jZS_4ci4.js → dataset-CqrqTYPz.js} +1 -1
- package/dist/{plugin-B3NxUCVn.js → plugin-DpncES1p.js} +73 -0
- package/dist/recall-bootstrap.js +3 -1
- package/dist/recall.config.schema.json +6 -0
- package/dist/recall.d.ts +22 -0
- package/dist/recall.js +2 -2
- package/dist/{row-ByLFE2Wx.d.ts → row-SRhura12.d.ts} +2 -0
- package/dist/sdk.d.ts +1 -1
- package/dist/{src-CkZw6xVD.js → src-C12EA0w9.js} +20 -2
- package/dist/substrate-admin.js +1 -1
- package/dist/substrate.js +107 -11
- package/package.json +19 -19
package/README.md
CHANGED
|
@@ -101,7 +101,7 @@ set -a && source .env && set +a
|
|
|
101
101
|
|
|
102
102
|
`substrate init` writes a `package.json` pinning `@mnmal-ai/substrate` and the chosen optional plugins, a `pnpm-workspace.yaml`, a `.env` with `SUBSTRATE_PLUGIN_DIR=.`, a `substrate-recall.config.json` for the embedder, and, unless `--store=memory`, a `docker-compose.yml` running `pgvector/pgvector:pg17`. Flags: `--store=memory|postgres`, `--port`, `--pg-port`, `--plugins=<csv>` (from `cortext`, `substrate-nl`, `substrate-sentinel`; none by default), `--embedder=bge-base|minilm|ollama|none` (default `bge-base`), `--tls --tls-cert-path --tls-key-path`, `--substrate-url`, `--yes`, `--force`.
|
|
103
103
|
|
|
104
|
-
`SUBSTRATE_PLUGIN_DIR` is the directory whose `node_modules/` holds the plugins; in a scaffolded runtime that is the runtime directory itself. Recall and proposals load from `@mnmal-ai/substrate` itself, so they are present whenever the binary is installed there. Kernel plugins (`substrate-recall`, `substrate-nl`) compose first, then coordinators (`substrate-proposals`, `substrate-sentinel`, …), alphabetical within each tier. Per-plugin overrides live at `<SUBSTRATE_PLUGIN_DIR>/<pkg-basename>.config.json`. With `SUBSTRATE_PLUGIN_DIR` unset the binary boots with only the `_system` admin namespace and `/health`.
|
|
104
|
+
`SUBSTRATE_PLUGIN_DIR` is the directory whose `node_modules/` holds the plugins; in a scaffolded runtime that is the runtime directory itself. Recall and proposals load from `@mnmal-ai/substrate` itself, so they are present whenever the binary is installed there. Kernel plugins (`substrate-recall`, `substrate-nl`) compose first, then coordinators (`substrate-proposals`, `substrate-sentinel`, …), alphabetical within each tier. Per-plugin overrides live at `<SUBSTRATE_PLUGIN_DIR>/<pkg-basename>.config.json`. With `SUBSTRATE_PLUGIN_DIR` unset the binary boots with only the `_system` admin namespace and `/health`. Set, it must hold a `package.json` listing the plugins under `dependencies`, installed into its `node_modules/`; a missing `package.json`, or listed plugins with no `node_modules/`, refuses boot with a message naming the fix.
|
|
105
105
|
|
|
106
106
|
### Adding a plugin
|
|
107
107
|
|
package/dist/client.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { E as Row, l as PackEnvelope, u as Schema } from "./schema-g4FQLr0V.js";
|
|
2
2
|
import { a as RuntimeErrors, i as RuntimeErrorEntry, o as RuntimeMetrics, r as ResolverStat, s as SlowRequest, t as ProcessStat } from "./runtime-metrics-DwpDJ8vz.js";
|
|
3
3
|
import { C as QualifiedFrame } from "./plugin-t2g2wX0b.js";
|
|
4
|
-
import { A as and, B as SYSTEM_NAMESPACE, C as flattenGrant, D as parseGrantEntries, E as intersectGrants, F as TokenProviderOpts, G as resolveAdminCredential, H as isActiveAdminKeyRow, I as adminClaimCapabilities, J as NlAskRequest, K as resolveDefaultKeyPath, L as isEncryptedKey, M as or, N as ADMIN_CAPABILITIES, O as decodeList, P as DEFAULT_CAPABILITIES, R as makeAuthorizationProvider, S as expandFlat, T as grantIsUniform, U as AdminCredential, V as describeGrant, W as AdminEnv, Y as NlAskResult, _ as newlyGrantedPairs, a as denyAll, b as capabilitiesFor, c as requireIdentity, d as KeyUpdateRefusal, f as checkRevocable, ft as NlWireEvent, g as newlyGranted, h as grantEntriesConfer, i as allowVerbs, j as not, k as encodeList, l as requireNamespace, lt as NlServedPack, m as grantCovers, n as row, o as denyVerbs, p as checkUpdatable, q as resolveDefaultKidPath, r as allowAll, rt as NlOutcome, s as requireCapability, t as applyRowFilters, u as KeyMutationRefusal, ut as NlTrailEvent, v as revokedAtIso, w as grantEntryError, x as encodeGrant, y as Grant, z as makeTokenProvider } from "./row-
|
|
4
|
+
import { A as and, B as SYSTEM_NAMESPACE, C as flattenGrant, D as parseGrantEntries, E as intersectGrants, F as TokenProviderOpts, G as resolveAdminCredential, H as isActiveAdminKeyRow, I as adminClaimCapabilities, J as NlAskRequest, K as resolveDefaultKeyPath, L as isEncryptedKey, M as or, N as ADMIN_CAPABILITIES, O as decodeList, P as DEFAULT_CAPABILITIES, R as makeAuthorizationProvider, S as expandFlat, T as grantIsUniform, U as AdminCredential, V as describeGrant, W as AdminEnv, Y as NlAskResult, _ as newlyGrantedPairs, a as denyAll, b as capabilitiesFor, c as requireIdentity, d as KeyUpdateRefusal, f as checkRevocable, ft as NlWireEvent, g as newlyGranted, h as grantEntriesConfer, i as allowVerbs, j as not, k as encodeList, l as requireNamespace, lt as NlServedPack, m as grantCovers, n as row, o as denyVerbs, p as checkUpdatable, q as resolveDefaultKidPath, r as allowAll, rt as NlOutcome, s as requireCapability, t as applyRowFilters, u as KeyMutationRefusal, ut as NlTrailEvent, v as revokedAtIso, w as grantEntryError, x as encodeGrant, y as Grant, z as makeTokenProvider } from "./row-SRhura12.js";
|
|
5
5
|
import { s as SchemaDriftError } from "./index-D7mSOFlM.js";
|
|
6
6
|
import { Signal } from "@preact/signals-core";
|
|
7
7
|
//#region ../../packages/substrate-client/src/admin/surgery.d.ts
|
|
@@ -11,7 +11,7 @@ import { createHash, createPublicKey, generateKeyPairSync, randomUUID } from "no
|
|
|
11
11
|
import postgres from "postgres";
|
|
12
12
|
|
|
13
13
|
//#region package.json
|
|
14
|
-
var version = "1.0.0-alpha.
|
|
14
|
+
var version = "1.0.0-alpha.9";
|
|
15
15
|
|
|
16
16
|
//#endregion
|
|
17
17
|
//#region src/admin/errors.ts
|
|
@@ -2805,6 +2805,50 @@ async function alignLocked(s, opts) {
|
|
|
2805
2805
|
};
|
|
2806
2806
|
}
|
|
2807
2807
|
|
|
2808
|
+
//#endregion
|
|
2809
|
+
//#region ../../plugins/substrate-recall/src/embed-backfill.ts
|
|
2810
|
+
async function runEmbedBackfill(deps) {
|
|
2811
|
+
const result = {
|
|
2812
|
+
embedded: 0,
|
|
2813
|
+
unindexable: 0,
|
|
2814
|
+
failed: 0
|
|
2815
|
+
};
|
|
2816
|
+
const staleRowIds = deps.store.staleRowIds?.bind(deps.store);
|
|
2817
|
+
if (staleRowIds === void 0) return result;
|
|
2818
|
+
for (const { namespace, type } of deps.registry.types()) {
|
|
2819
|
+
const reembed = deps.registry.get(namespace, type)?.reembed;
|
|
2820
|
+
const dataset = deps.resolveDataset(namespace);
|
|
2821
|
+
if (reembed === void 0 || dataset === null) continue;
|
|
2822
|
+
const key = `${namespace}/${type}`;
|
|
2823
|
+
const ids = await staleRowIds({
|
|
2824
|
+
namespace,
|
|
2825
|
+
type,
|
|
2826
|
+
afterId: deps.cursors.get(key) ?? "",
|
|
2827
|
+
limit: deps.perTypeLimit
|
|
2828
|
+
});
|
|
2829
|
+
if (ids.length < deps.perTypeLimit) deps.cursors.delete(key);
|
|
2830
|
+
else deps.cursors.set(key, ids[ids.length - 1]);
|
|
2831
|
+
if (ids.length === 0) continue;
|
|
2832
|
+
const { rows } = await dataset.read(type, {
|
|
2833
|
+
ids,
|
|
2834
|
+
limit: ids.length
|
|
2835
|
+
});
|
|
2836
|
+
for (const row of rows) {
|
|
2837
|
+
const outcome = await reembed(row);
|
|
2838
|
+
if (outcome.status === "embedded") result.embedded++;
|
|
2839
|
+
else if (outcome.status === "unindexable") result.unindexable++;
|
|
2840
|
+
else if (outcome.status === "failed") {
|
|
2841
|
+
result.failed++;
|
|
2842
|
+
if (outcome.code === "unavailable") return {
|
|
2843
|
+
...result,
|
|
2844
|
+
stopped: outcome.error
|
|
2845
|
+
};
|
|
2846
|
+
}
|
|
2847
|
+
}
|
|
2848
|
+
}
|
|
2849
|
+
return result;
|
|
2850
|
+
}
|
|
2851
|
+
|
|
2808
2852
|
//#endregion
|
|
2809
2853
|
//#region ../../plugins/substrate-recall/src/expansion-resolver.ts
|
|
2810
2854
|
function makeExpansionResolver(registry, edges, cfg) {
|
|
@@ -3654,6 +3698,9 @@ function makeSpreadResolver(registry, edges) {
|
|
|
3654
3698
|
//#endregion
|
|
3655
3699
|
//#region ../../plugins/substrate-recall/src/plugin.ts
|
|
3656
3700
|
const DEFAULT_CLUSTER_DIMENSION = 768;
|
|
3701
|
+
const DEFAULT_EMBED_BACKFILL_INTERVAL_MS = 6e4;
|
|
3702
|
+
/** Rows per embedded type per pass, so one pass stays a few seconds of embed work. */
|
|
3703
|
+
const EMBED_BACKFILL_PER_TYPE = 50;
|
|
3657
3704
|
const DEFAULT_QUERY_EMBED_TIMEOUT_MS = 3e4;
|
|
3658
3705
|
const DEFAULT_WRITE_EMBED_TIMEOUT_MS = 3e4;
|
|
3659
3706
|
/** `embedder` with `timeoutMs` applied to every call that does not set its own. */
|
|
@@ -3736,6 +3783,7 @@ function createRecallPlugin(cfg) {
|
|
|
3736
3783
|
contractVersion: ">=1.0.0-alpha.0 <2.0.0",
|
|
3737
3784
|
async onKernelReady(ctx) {
|
|
3738
3785
|
if (cfg.embedder === void 0 || ctx.sql === null) return;
|
|
3786
|
+
let dimensionMismatch = false;
|
|
3739
3787
|
const report = await alignVectorDimensions(ctx.sql, {
|
|
3740
3788
|
dimension: cfg.embedder.dimension,
|
|
3741
3789
|
...cfg.migration?.hnswM !== void 0 ? { hnswM: cfg.migration.hnswM } : {},
|
|
@@ -3743,6 +3791,7 @@ function createRecallPlugin(cfg) {
|
|
|
3743
3791
|
});
|
|
3744
3792
|
for (const [column, state] of [["embeddings.embedding", report.embeddings], ["recall_clusters.centroid", report.centroids]]) {
|
|
3745
3793
|
if (state !== "populated") continue;
|
|
3794
|
+
dimensionMismatch = true;
|
|
3746
3795
|
const found = column.startsWith("embeddings") ? report.found.embeddings : report.found.centroids;
|
|
3747
3796
|
ctx.metric("recall_dimension_mismatch", {
|
|
3748
3797
|
column,
|
|
@@ -3752,6 +3801,30 @@ function createRecallPlugin(cfg) {
|
|
|
3752
3801
|
});
|
|
3753
3802
|
process.emitWarning(`${column} is vector(${found}) and holds rows, but the embedder is ${cfg.embedder.modelId} at ${cfg.embedder.dimension}. Writes to it will fail until the rows are cleared (they are derived) and re-embedded: delete them, restart, then POST /substrate/v1/embed/reindex.`, { code: "RECALL_DIMENSION_MISMATCH" });
|
|
3754
3803
|
}
|
|
3804
|
+
const intervalMs = cfg.embedBackfillIntervalMs ?? DEFAULT_EMBED_BACKFILL_INTERVAL_MS;
|
|
3805
|
+
const resolveDataset = cfg.resolveDataset;
|
|
3806
|
+
if (dimensionMismatch || intervalMs === 0 || resolveDataset === void 0) return;
|
|
3807
|
+
const cursors = /* @__PURE__ */ new Map();
|
|
3808
|
+
let running = false;
|
|
3809
|
+
setInterval(() => {
|
|
3810
|
+
if (running) return;
|
|
3811
|
+
running = true;
|
|
3812
|
+
runEmbedBackfill({
|
|
3813
|
+
registry: embedSourceRegistry,
|
|
3814
|
+
store: cfg.store,
|
|
3815
|
+
resolveDataset,
|
|
3816
|
+
cursors,
|
|
3817
|
+
perTypeLimit: EMBED_BACKFILL_PER_TYPE
|
|
3818
|
+
}).then((r) => {
|
|
3819
|
+
if (r.embedded + r.failed === 0 && r.stopped === void 0) return;
|
|
3820
|
+
ctx.metric("recall_embed_backfill", r);
|
|
3821
|
+
process.stdout.write(`recall.embed_backfill ${JSON.stringify(r)}\n`);
|
|
3822
|
+
}).catch((err) => {
|
|
3823
|
+
process.stderr.write(`recall.embed_backfill failed: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
3824
|
+
}).finally(() => {
|
|
3825
|
+
running = false;
|
|
3826
|
+
});
|
|
3827
|
+
}, intervalMs).unref();
|
|
3755
3828
|
},
|
|
3756
3829
|
migrations: [
|
|
3757
3830
|
embeddingsMigration(cfg.migration ?? {}),
|
package/dist/recall-bootstrap.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
|
|
2
2
|
import { Pt as defineBootstrap } from "./src-DjIuRjzF.js";
|
|
3
3
|
import { t as FakeEmbedder } from "./fake-DUVtKVSE.js";
|
|
4
|
-
import { t as createRecallPlugin } from "./plugin-
|
|
4
|
+
import { t as createRecallPlugin } from "./plugin-DpncES1p.js";
|
|
5
5
|
import { z } from "zod";
|
|
6
6
|
|
|
7
7
|
//#region ../../packages/substrate-embedder/src/types.ts
|
|
@@ -384,6 +384,7 @@ const RecallBootstrapConfigSchema = z.object({
|
|
|
384
384
|
embedder: EmbedderSchema.optional(),
|
|
385
385
|
queryEmbedTimeoutMs: z.number().int().positive().default(3e4),
|
|
386
386
|
writeEmbedTimeoutMs: z.number().int().positive().default(3e4),
|
|
387
|
+
embedBackfillIntervalMs: z.number().int().nonnegative().default(6e4),
|
|
387
388
|
hnswM: z.number().int().positive().default(16),
|
|
388
389
|
hnswEfConstruction: z.number().int().positive().default(64),
|
|
389
390
|
salience: SalienceSchema.default({
|
|
@@ -433,6 +434,7 @@ const bootstrap = defineBootstrap.kernel({
|
|
|
433
434
|
...embedder !== null ? { embedder } : {},
|
|
434
435
|
queryEmbedTimeoutMs: config.queryEmbedTimeoutMs,
|
|
435
436
|
writeEmbedTimeoutMs: config.writeEmbedTimeoutMs,
|
|
437
|
+
embedBackfillIntervalMs: config.embedBackfillIntervalMs,
|
|
436
438
|
...deps.lexicalStore !== void 0 ? { lexicalStore: deps.lexicalStore } : {},
|
|
437
439
|
store: deps.store,
|
|
438
440
|
...deps.stats !== void 0 ? { stats: deps.stats } : {},
|
|
@@ -124,6 +124,12 @@
|
|
|
124
124
|
"exclusiveMinimum": 0,
|
|
125
125
|
"maximum": 9007199254740991
|
|
126
126
|
},
|
|
127
|
+
"embedBackfillIntervalMs": {
|
|
128
|
+
"default": 60000,
|
|
129
|
+
"type": "integer",
|
|
130
|
+
"minimum": 0,
|
|
131
|
+
"maximum": 9007199254740991
|
|
132
|
+
},
|
|
127
133
|
"hnswM": {
|
|
128
134
|
"default": 16,
|
|
129
135
|
"type": "integer",
|
package/dist/recall.d.ts
CHANGED
|
@@ -305,6 +305,17 @@ interface EmbeddingsStore {
|
|
|
305
305
|
* mismatched vector surfaces as a substrate-level error.
|
|
306
306
|
*/
|
|
307
307
|
upsert(input: EmbeddingsStoreUpsertInput): Promise<void>;
|
|
308
|
+
/**
|
|
309
|
+
* Ids of live rows of (namespace, type) whose vector is missing or older than
|
|
310
|
+
* the row, after `afterId` in id order: the writes whose embed failed. Optional;
|
|
311
|
+
* a store that cannot see the rows table omits it and nothing is backfilled.
|
|
312
|
+
*/
|
|
313
|
+
staleRowIds?(input: {
|
|
314
|
+
namespace: string;
|
|
315
|
+
type: string;
|
|
316
|
+
afterId: string;
|
|
317
|
+
limit: number;
|
|
318
|
+
}): Promise<string[]>;
|
|
308
319
|
/**
|
|
309
320
|
* Remove the embedding for one row, leaving the row itself untouched. Used
|
|
310
321
|
* to un-index a row whose `embed.source` returns the null sentinel.
|
|
@@ -1704,6 +1715,12 @@ export declare class PgEmbeddingsStore implements EmbeddingsStore {
|
|
|
1704
1715
|
*
|
|
1705
1716
|
* @returns number of orphaned embeddings deleted.
|
|
1706
1717
|
*/
|
|
1718
|
+
staleRowIds(input: {
|
|
1719
|
+
namespace: string;
|
|
1720
|
+
type: string;
|
|
1721
|
+
afterId: string;
|
|
1722
|
+
limit: number;
|
|
1723
|
+
}): Promise<string[]>;
|
|
1707
1724
|
purgeOrphans(): Promise<number>;
|
|
1708
1725
|
}
|
|
1709
1726
|
//#endregion
|
|
@@ -1758,6 +1775,11 @@ interface RecallPluginConfig {
|
|
|
1758
1775
|
* the row's previous vector until its next write or a reindex. Default 30000.
|
|
1759
1776
|
*/
|
|
1760
1777
|
writeEmbedTimeoutMs?: number;
|
|
1778
|
+
/**
|
|
1779
|
+
* How often to re-embed rows whose write-path embed failed or timed out (their
|
|
1780
|
+
* vector is missing or older than the row). 0 disables. Default 60000.
|
|
1781
|
+
*/
|
|
1782
|
+
embedBackfillIntervalMs?: number;
|
|
1761
1783
|
store: EmbeddingsStore;
|
|
1762
1784
|
/**
|
|
1763
1785
|
* Store for the lexical arm. Absent → types declaring `lexical.source`
|
package/dist/recall.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
|
|
2
|
-
import { a as InMemoryLexicalStore, i as PgClusterStore, n as PgLexicalStore, o as InMemoryEmbeddingsStore, r as PgEmbeddingsStore, s as InMemoryClusterStore, t as RECALL_VERSION } from "./src-
|
|
3
|
-
import { $ as CLUSTERER_BACKEND_VERSION, A as CLUSTER_MEMBER_FIELDS, C as EmbedSourceRegistry, D as makeReembed, E as makeEmbedPostWrite, F as unknownParams, G as DEFAULT_CRYSTALLISE_CONFIG, H as CRYSTALLISE_IN_FLIGHT_CODE, I as handleRecallRequest, J as runClusterer, K as handleCrystallise, L as RRF_K, M as recallRouteHandler, N as checkParams, O as crystalliseRouteHandler, P as unknownParamMessage, Q as CLUSTERER_BACKEND, R as armRanks, S as createEmbedStats, T as embedRowReporting, U as createCrystalliseGuard, X as l2normalize, _ as EMBEDDINGS_FETCH_MAX_REFS, a as REINDEX_MAX_LIMIT, b as handleLexicalCoverage, c as SALIENCE_BOOST_MAX_TARGETS, d as embeddingsMigration, et as memberKey, f as recallMcpTool, g as embeddingsFetchRouteHandler, h as embeddingSimilarOperator, i as REINDEX_DEFAULT_LIMIT, j as buildClusterSurface, k as CLUSTER_FIELDS, l as lexicalMigration, m as makeLexicalPostWrite, n as salienceBoostRouteHandler, nt as validateSelector, o as handleReindex, p as makeLexicalClassify, q as validateCrystalliseInput, r as reindexRouteHandler, s as validateReindexInput, t as createRecallPlugin, tt as SELECT_CANDIDATES_MAX_LIMIT, u as recallClustersMigration, v as embedStatsRouteHandler, w as embedAndStore, x as validateLexicalCoverageInput, y as handleEmbedStatsRequest, z as fuseByRank } from "./plugin-
|
|
2
|
+
import { a as InMemoryLexicalStore, i as PgClusterStore, n as PgLexicalStore, o as InMemoryEmbeddingsStore, r as PgEmbeddingsStore, s as InMemoryClusterStore, t as RECALL_VERSION } from "./src-C12EA0w9.js";
|
|
3
|
+
import { $ as CLUSTERER_BACKEND_VERSION, A as CLUSTER_MEMBER_FIELDS, C as EmbedSourceRegistry, D as makeReembed, E as makeEmbedPostWrite, F as unknownParams, G as DEFAULT_CRYSTALLISE_CONFIG, H as CRYSTALLISE_IN_FLIGHT_CODE, I as handleRecallRequest, J as runClusterer, K as handleCrystallise, L as RRF_K, M as recallRouteHandler, N as checkParams, O as crystalliseRouteHandler, P as unknownParamMessage, Q as CLUSTERER_BACKEND, R as armRanks, S as createEmbedStats, T as embedRowReporting, U as createCrystalliseGuard, X as l2normalize, _ as EMBEDDINGS_FETCH_MAX_REFS, a as REINDEX_MAX_LIMIT, b as handleLexicalCoverage, c as SALIENCE_BOOST_MAX_TARGETS, d as embeddingsMigration, et as memberKey, f as recallMcpTool, g as embeddingsFetchRouteHandler, h as embeddingSimilarOperator, i as REINDEX_DEFAULT_LIMIT, j as buildClusterSurface, k as CLUSTER_FIELDS, l as lexicalMigration, m as makeLexicalPostWrite, n as salienceBoostRouteHandler, nt as validateSelector, o as handleReindex, p as makeLexicalClassify, q as validateCrystalliseInput, r as reindexRouteHandler, s as validateReindexInput, t as createRecallPlugin, tt as SELECT_CANDIDATES_MAX_LIMIT, u as recallClustersMigration, v as embedStatsRouteHandler, w as embedAndStore, x as validateLexicalCoverageInput, y as handleEmbedStatsRequest, z as fuseByRank } from "./plugin-DpncES1p.js";
|
|
4
4
|
|
|
5
5
|
export { CLUSTERER_BACKEND, CLUSTERER_BACKEND_VERSION, CLUSTER_FIELDS, CLUSTER_MEMBER_FIELDS, CRYSTALLISE_IN_FLIGHT_CODE, DEFAULT_CRYSTALLISE_CONFIG, EMBEDDINGS_FETCH_MAX_REFS, EmbedSourceRegistry, InMemoryClusterStore, InMemoryEmbeddingsStore, InMemoryLexicalStore, PgClusterStore, PgEmbeddingsStore, PgLexicalStore, RECALL_VERSION, REINDEX_DEFAULT_LIMIT, REINDEX_MAX_LIMIT, RRF_K, SALIENCE_BOOST_MAX_TARGETS, SELECT_CANDIDATES_MAX_LIMIT, armRanks, buildClusterSurface, checkParams, createCrystalliseGuard, createEmbedStats, createRecallPlugin, crystalliseRouteHandler, embedAndStore, embedRowReporting, embedStatsRouteHandler, embeddingSimilarOperator, embeddingsFetchRouteHandler, embeddingsMigration, fuseByRank, handleCrystallise, handleEmbedStatsRequest, handleLexicalCoverage, handleRecallRequest, handleReindex, l2normalize, lexicalMigration, makeEmbedPostWrite, makeLexicalClassify, makeLexicalPostWrite, makeReembed, memberKey, recallClustersMigration, recallMcpTool, recallRouteHandler, reindexRouteHandler, runClusterer, salienceBoostRouteHandler, unknownParamMessage, unknownParams, validateCrystalliseInput, validateLexicalCoverageInput, validateReindexInput, validateSelector };
|
|
@@ -82,6 +82,8 @@ type NlOutcome = {
|
|
|
82
82
|
answer: string;
|
|
83
83
|
/** Absent on the wire when the engine rejected the frame; `errors` then says why. */
|
|
84
84
|
data: unknown;
|
|
85
|
+
/** On a `count` answer: every row the frame matches, across pages; `data` holds only the first page. Absent when counting stopped short, with `errors` saying why. */
|
|
86
|
+
count?: number;
|
|
85
87
|
errors?: unknown[];
|
|
86
88
|
source: 'parser' | 'judge' | 'handoff';
|
|
87
89
|
handoff?: string;
|
package/dist/sdk.d.ts
CHANGED
|
@@ -3,7 +3,7 @@ import { a as RuntimeErrors, c as BridgeClient, d as BridgeFrame, i as RuntimeEr
|
|
|
3
3
|
import { $ as PolicyProbeContext, A as ScheduledTask, B as StartOpts, C as QualifiedFrame, D as RegistryEntryInput, E as RegistryEntry, F as HttpMethod, G as WsConnection, H as SubstrateResponse, I as PluginRouter, J as AutoCrudVerb, K as WsHandler, L as RequestObservation, M as VirtualMutationDef, N as VirtualTypeDef, O as RegistryEntryKind, P as HttpHandler, Q as PolicyDecision, R as RequestObserver, S as PostWriteHook, St as WriteResult, T as RegistryBase, U as SubstrateServer, V as SubstrateRequest, W as TlsConfig, X as Obligation, Y as Identity, Z as Policy, _ as NamespaceDefinition, _t as ReadResult, a as EngineHandle, at as TypedResolver, b as PluginLifecycleContext, bt as WhereClause, c as FrameEntry, ct as Dataset, d as KernelMigrationContext, dt as DropResult, et as RequestContext, f as KernelNamespaceContribution, ft as FieldKind, g as NamespaceConfig, gt as ReadParams, h as McpToolRegistration, ht as OrderByClause, i as EncodeContext, it as RowFilterObligation, j as ScheduledTaskContext, k as RegistryEntryResolver, l as KernelLifecycleContext, lt as DatasetRegistry, m as KernelRouteClaim, mt as FilterCapabilities, n as DropParticipant, nt as ResolverArgs, o as EngineInput, ot as WriteEvent, p as KernelPlugin, pt as FieldPredicate, q as AfterTxWork, r as EmbedSourceContext, rt as ResolverReturn, s as FilterOperatorDef, st as ConditionalUpdateResult, t as CoordinatorPlugin, tt as Resolver, u as KernelMigration, ut as DeleteResult, v as NamespaceInstallContext, vt as TxScopeHandles, w as Registry, x as PluginRouteContext, xt as WriteOptions, y as NamespaceTypeDef, yt as UpdateFieldRef, z as RuntimeOp } from "./plugin-t2g2wX0b.js";
|
|
4
4
|
import { a as ResolverFrameError, c as isAuthCause, i as ReservedField, l as isResolverFrameError, n as MissingReferentError, o as RestrictViolation, r as NamespaceMismatch, s as UnsupportedFilterError, t as BridgeUnreachableError, u as isRestrictViolation } from "./errors-D4vKuwz8.js";
|
|
5
5
|
import { a as WsResponseFrame, c as ResolveTarget, d as TargetState, f as IdempotencyStore, i as WsRequestFrame, l as SweepDeps, n as WS_REASON_SERVER_RESTART, o as WsResponseKind, r as WS_REASON_SUBSCRIPTION_OVERLOADED, s as BridgeRegistry, t as WS_CODE_SUBSCRIPTION_UNSUPPORTED_FRAME, u as SweepState } from "./ws-C72iVS5T.js";
|
|
6
|
-
import { $ as NlJudgeStep, A as and, B as SYSTEM_NAMESPACE, C as flattenGrant, D as parseGrantEntries, E as intersectGrants, F as TokenProviderOpts, G as resolveAdminCredential, H as isActiveAdminKeyRow, I as adminClaimCapabilities, J as NlAskRequest, K as resolveDefaultKeyPath, L as isEncryptedKey, M as or, N as ADMIN_CAPABILITIES, O as decodeList, P as DEFAULT_CAPABILITIES, Q as NlJudgeChoice, R as makeAuthorizationProvider, S as expandFlat, T as grantIsUniform, U as AdminCredential, V as describeGrant, W as AdminEnv, X as NlEvidence, Y as NlAskResult, Z as NlJudgeAction, _ as newlyGrantedPairs, a as denyAll, at as NlPackType, b as capabilitiesFor, c as requireIdentity, ct as NlParsedData, d as KeyUpdateRefusal, dt as NlValueAlias, et as NlJudgedStatus, f as checkRevocable, ft as NlWireEvent, g as newlyGranted, gt as Query, h as grantEntriesConfer, ht as NLVocabularyHint, i as allowVerbs, it as NlPackField, j as not, k as encodeList, l as requireNamespace, lt as NlServedPack, m as grantCovers, mt as NLResult, n as row, nt as NlNamespaceRulings, o as denyVerbs, ot as NlPackValueRef, p as checkUpdatable, pt as NLBase, q as resolveDefaultKidPath, r as allowAll, rt as NlOutcome, s as requireCapability, st as NlParseStatus, t as applyRowFilters, tt as NlNamespacePack, u as KeyMutationRefusal, ut as NlTrailEvent, v as revokedAtIso, w as grantEntryError, x as encodeGrant, y as Grant, z as makeTokenProvider } from "./row-
|
|
6
|
+
import { $ as NlJudgeStep, A as and, B as SYSTEM_NAMESPACE, C as flattenGrant, D as parseGrantEntries, E as intersectGrants, F as TokenProviderOpts, G as resolveAdminCredential, H as isActiveAdminKeyRow, I as adminClaimCapabilities, J as NlAskRequest, K as resolveDefaultKeyPath, L as isEncryptedKey, M as or, N as ADMIN_CAPABILITIES, O as decodeList, P as DEFAULT_CAPABILITIES, Q as NlJudgeChoice, R as makeAuthorizationProvider, S as expandFlat, T as grantIsUniform, U as AdminCredential, V as describeGrant, W as AdminEnv, X as NlEvidence, Y as NlAskResult, Z as NlJudgeAction, _ as newlyGrantedPairs, a as denyAll, at as NlPackType, b as capabilitiesFor, c as requireIdentity, ct as NlParsedData, d as KeyUpdateRefusal, dt as NlValueAlias, et as NlJudgedStatus, f as checkRevocable, ft as NlWireEvent, g as newlyGranted, gt as Query, h as grantEntriesConfer, ht as NLVocabularyHint, i as allowVerbs, it as NlPackField, j as not, k as encodeList, l as requireNamespace, lt as NlServedPack, m as grantCovers, mt as NLResult, n as row, nt as NlNamespaceRulings, o as denyVerbs, ot as NlPackValueRef, p as checkUpdatable, pt as NLBase, q as resolveDefaultKidPath, r as allowAll, rt as NlOutcome, s as requireCapability, st as NlParseStatus, t as applyRowFilters, tt as NlNamespacePack, u as KeyMutationRefusal, ut as NlTrailEvent, v as revokedAtIso, w as grantEntryError, x as encodeGrant, y as Grant, z as makeTokenProvider } from "./row-SRhura12.js";
|
|
7
7
|
//#region ../../packages/substrate-types/src/assert-never.d.ts
|
|
8
8
|
/**
|
|
9
9
|
* Exhaustiveness guard for the `default:` arm of a `switch` over a
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
|
|
2
2
|
import { n as txScopeIdFor } from "./src-BD27zeGF.js";
|
|
3
|
-
import { B as ROW_FILTER_FIELD, V as matchesRowFilter, W as inFlightError, Y as unresolvedRow, Z as cosineSimilarity, et as memberKey } from "./plugin-
|
|
3
|
+
import { B as ROW_FILTER_FIELD, V as matchesRowFilter, W as inFlightError, Y as unresolvedRow, Z as cosineSimilarity, et as memberKey } from "./plugin-DpncES1p.js";
|
|
4
4
|
|
|
5
5
|
//#region ../../plugins/substrate-recall/package.json
|
|
6
|
-
var version = "1.0.0-alpha.
|
|
6
|
+
var version = "1.0.0-alpha.9";
|
|
7
7
|
|
|
8
8
|
//#endregion
|
|
9
9
|
//#region ../../plugins/substrate-recall/src/in-memory-cluster-store.ts
|
|
@@ -870,6 +870,24 @@ var PgEmbeddingsStore = class {
|
|
|
870
870
|
*
|
|
871
871
|
* @returns number of orphaned embeddings deleted.
|
|
872
872
|
*/
|
|
873
|
+
async staleRowIds(input) {
|
|
874
|
+
return (await this.#sql`
|
|
875
|
+
SELECT r.id FROM rows r
|
|
876
|
+
WHERE r.namespace = ${input.namespace}
|
|
877
|
+
AND r.type = ${input.type}
|
|
878
|
+
AND r.deleted_at IS NULL
|
|
879
|
+
AND r.id > ${input.afterId}
|
|
880
|
+
AND NOT EXISTS (
|
|
881
|
+
SELECT 1 FROM embeddings e
|
|
882
|
+
WHERE e.row_namespace = r.namespace
|
|
883
|
+
AND e.row_type = r.type
|
|
884
|
+
AND e.row_id = r.id
|
|
885
|
+
AND (r.updated_at IS NULL OR e.updated_at >= r.updated_at)
|
|
886
|
+
)
|
|
887
|
+
ORDER BY r.id
|
|
888
|
+
LIMIT ${input.limit}
|
|
889
|
+
`).map((r) => r.id);
|
|
890
|
+
}
|
|
873
891
|
async purgeOrphans() {
|
|
874
892
|
return (await this.#sql`
|
|
875
893
|
DELETE FROM embeddings e
|
package/dist/substrate-admin.js
CHANGED
|
@@ -4,7 +4,7 @@ import { d as buildAuditRow, f as AUDIT_EVENT_TYPE } from "./src-DjIuRjzF.js";
|
|
|
4
4
|
import { d as resolveDefaultKidPath, u as resolveDefaultKeyPath } from "./env-utils-CGNazUNt.js";
|
|
5
5
|
import { E as describeGrant, I as decodeList, L as encodeList, M as grantEntryError, g as checkUpdatable, h as checkRevocable, y as newlyGranted } from "./row-DZ0Fnpc6.js";
|
|
6
6
|
import { w as reconcileIdentities } from "./src-DiNTSaFk.js";
|
|
7
|
-
import { A as describeAdminTarget, B as version, C as CLI_ACTOR_KID, D as listActiveAdminKeys, E as generateAdminKeypair, F as RESTART_NOTE, I as USAGE, L as argvError, M as resolveAdminTarget, N as ARGV_SPEC, O as readEd25519PublicKey, P as validateArgv, S as BootstrapError, T as forUpdate, _ as REVOKE_EXIT_CODES, a as formatRegisterOutput, b as runKeyPatchViaRoute, c as runRegisterViaRoute, d as formatDoctorReport, f as runDoctor, g as bootstrapKidPath, h as runBootstrap, i as runOnboard, j as resolveAdminAuth, k as readKeyById, l as grantLines, m as parseBootstrapArgs, n as formatChecklist, o as parseRegisterArgs, p as formatBootstrapOutput, r as onboardCommand, s as runRegister, t as openPgDataset, u as doctorCommand, v as UPDATE_EXIT_CODES, w as base64Url, x as ACTOR_HINT, y as runKeyListViaRoute, z as AdminCliError } from "./dataset-
|
|
7
|
+
import { A as describeAdminTarget, B as version, C as CLI_ACTOR_KID, D as listActiveAdminKeys, E as generateAdminKeypair, F as RESTART_NOTE, I as USAGE, L as argvError, M as resolveAdminTarget, N as ARGV_SPEC, O as readEd25519PublicKey, P as validateArgv, S as BootstrapError, T as forUpdate, _ as REVOKE_EXIT_CODES, a as formatRegisterOutput, b as runKeyPatchViaRoute, c as runRegisterViaRoute, d as formatDoctorReport, f as runDoctor, g as bootstrapKidPath, h as runBootstrap, i as runOnboard, j as resolveAdminAuth, k as readKeyById, l as grantLines, m as parseBootstrapArgs, n as formatChecklist, o as parseRegisterArgs, p as formatBootstrapOutput, r as onboardCommand, s as runRegister, t as openPgDataset, u as doctorCommand, v as UPDATE_EXIT_CODES, w as base64Url, x as ACTOR_HINT, y as runKeyListViaRoute, z as AdminCliError } from "./dataset-CqrqTYPz.js";
|
|
8
8
|
import { existsSync, mkdirSync, realpathSync, rmSync, writeFileSync } from "node:fs";
|
|
9
9
|
import { dirname } from "node:path";
|
|
10
10
|
import { createHash, randomUUID } from "node:crypto";
|
package/dist/substrate.js
CHANGED
|
@@ -7,8 +7,8 @@ import { a as SubstrateClient, d as NetworkError, i as tokenAudience, n as resol
|
|
|
7
7
|
import { n as runScript, t as DEFAULT_MAX_WRITES } from "./script-C6DSa_Na.js";
|
|
8
8
|
import { a as runDerivedMigrations, o as runMigrations, r as deriveListIndexMigration, s as tryWithPgAdvisoryLock, t as PgDataset } from "./src-BD27zeGF.js";
|
|
9
9
|
import { t as PgEdgesStore } from "./pg-edges-store-KH_FQCxx.js";
|
|
10
|
-
import { a as InMemoryLexicalStore, i as PgClusterStore, n as PgLexicalStore, o as InMemoryEmbeddingsStore, r as PgEmbeddingsStore, s as InMemoryClusterStore } from "./src-
|
|
11
|
-
import { B as version, R as openSql, t as openPgDataset, z as AdminCliError } from "./dataset-
|
|
10
|
+
import { a as InMemoryLexicalStore, i as PgClusterStore, n as PgLexicalStore, o as InMemoryEmbeddingsStore, r as PgEmbeddingsStore, s as InMemoryClusterStore } from "./src-C12EA0w9.js";
|
|
11
|
+
import { B as version, R as openSql, t as openPgDataset, z as AdminCliError } from "./dataset-CqrqTYPz.js";
|
|
12
12
|
import { chmodSync, existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
|
|
13
13
|
import { homedir, hostname } from "node:os";
|
|
14
14
|
import { basename, dirname, join, resolve } from "node:path";
|
|
@@ -2622,24 +2622,29 @@ var PluginManifestError = class extends Error {
|
|
|
2622
2622
|
}
|
|
2623
2623
|
};
|
|
2624
2624
|
async function listPluginManifests(pluginDir) {
|
|
2625
|
+
const declared = await declaredDependencies(pluginDir);
|
|
2626
|
+
if (declared.size === 0) return [];
|
|
2625
2627
|
const nodeModulesDir = resolve(pluginDir, "node_modules");
|
|
2626
|
-
if (!await isDirectory(nodeModulesDir))
|
|
2627
|
-
const packageDirs = await listPackageDirs(nodeModulesDir,
|
|
2628
|
+
if (!await isDirectory(nodeModulesDir)) throw new PluginManifestError(void 0, `${pluginDir}/package.json lists ${[...declared].join(", ")} but there is no node_modules: run \`pnpm install\` in ${pluginDir}`);
|
|
2629
|
+
const packageDirs = await listPackageDirs(nodeModulesDir, declared);
|
|
2628
2630
|
const manifests = [];
|
|
2629
2631
|
for (const packageDir of packageDirs) manifests.push(...await readManifestsIfPresent(packageDir));
|
|
2632
|
+
if (manifests.length === 0) process.stderr.write(`substrate: SUBSTRATE_PLUGIN_DIR=${pluginDir} lists ${[...declared].join(", ")}, none of which is an installed substrate plugin; booting without plugins\n`);
|
|
2630
2633
|
return manifests;
|
|
2631
2634
|
}
|
|
2632
2635
|
/**
|
|
2633
2636
|
* The names `<pluginDir>/package.json` lists under `dependencies`. Only these
|
|
2634
2637
|
* are plugin candidates: a transitive package hoisted into `node_modules` is
|
|
2635
|
-
* not something the operator chose to run.
|
|
2638
|
+
* not something the operator chose to run. A missing package.json refuses
|
|
2639
|
+
* boot, since a plugin dir that can name no plugin is a misconfiguration.
|
|
2636
2640
|
*/
|
|
2637
2641
|
async function declaredDependencies(pluginDir) {
|
|
2638
2642
|
let raw;
|
|
2639
2643
|
try {
|
|
2640
2644
|
raw = await readFile(resolve(pluginDir, "package.json"), "utf8");
|
|
2641
|
-
} catch {
|
|
2642
|
-
|
|
2645
|
+
} catch (err) {
|
|
2646
|
+
if (err.code !== "ENOENT") throw err;
|
|
2647
|
+
throw new PluginManifestError(void 0, `SUBSTRATE_PLUGIN_DIR=${pluginDir} has no package.json, so no plugin can load. List each plugin under "dependencies" there (\`substrate init\` writes one) and run \`pnpm install\`, or unset SUBSTRATE_PLUGIN_DIR to boot without plugins`);
|
|
2643
2648
|
}
|
|
2644
2649
|
let parsed;
|
|
2645
2650
|
try {
|
|
@@ -4641,6 +4646,78 @@ function makeAgentJwtProvider(env, audience) {
|
|
|
4641
4646
|
return () => provider();
|
|
4642
4647
|
}
|
|
4643
4648
|
|
|
4649
|
+
//#endregion
|
|
4650
|
+
//#region src/mcp/schema-overview.ts
|
|
4651
|
+
const NAME = /^[A-Za-z_][A-Za-z0-9_-]{0,63}$/;
|
|
4652
|
+
const ENUM_VALUE = /^[A-Za-z0-9_./-]{1,40}$/;
|
|
4653
|
+
const MAX_ENUM_VALUES = 20;
|
|
4654
|
+
const MAX_ENUM_TEXT = 200;
|
|
4655
|
+
function safe(text, pattern) {
|
|
4656
|
+
return typeof text === "string" && pattern.test(text) ? text : "<unsafe>";
|
|
4657
|
+
}
|
|
4658
|
+
function enumText(values) {
|
|
4659
|
+
const shown = [];
|
|
4660
|
+
let length = 0;
|
|
4661
|
+
for (const v of values.slice(0, MAX_ENUM_VALUES)) {
|
|
4662
|
+
const text = safe(v, ENUM_VALUE);
|
|
4663
|
+
if (length + text.length > MAX_ENUM_TEXT) break;
|
|
4664
|
+
shown.push(text);
|
|
4665
|
+
length += text.length + 1;
|
|
4666
|
+
}
|
|
4667
|
+
return [...shown, ...shown.length < values.length ? ["…"] : []].join("|");
|
|
4668
|
+
}
|
|
4669
|
+
function fieldType(f, ns) {
|
|
4670
|
+
let t;
|
|
4671
|
+
if (f.kind === "list" && f.of !== void 0) t = `${fieldType(f.of, ns)}[]`;
|
|
4672
|
+
else if (f.kind === "crossRef") t = `→${f.ns === ns || f.ns === void 0 ? "" : `${safe(f.ns, NAME)}/`}${safe(f.type, NAME)}`;
|
|
4673
|
+
else if (f.kind === "struct") t = `{${Object.keys(f.fields ?? {}).map((k) => safe(k, NAME)).join(", ")}}`;
|
|
4674
|
+
else if (Array.isArray(f.enum)) t = f.nullable ? `(${enumText(f.enum)})` : enumText(f.enum);
|
|
4675
|
+
else t = safe(f.type ?? f.kind, NAME);
|
|
4676
|
+
const tags = [];
|
|
4677
|
+
if (f.indexed === false) tags.push("unindexed");
|
|
4678
|
+
if (f.readonly === true || f.serverSet !== void 0 && f.serverSet !== false) tags.push("readonly");
|
|
4679
|
+
return `${t}${f.nullable ? "?" : ""}${tags.length > 0 ? ` [${tags.join(", ")}]` : ""}`;
|
|
4680
|
+
}
|
|
4681
|
+
const AUTO_CRUD = /^(get|create|update|delete)(.+)$/;
|
|
4682
|
+
function schemaOverview(federated) {
|
|
4683
|
+
const out = {};
|
|
4684
|
+
for (const [ns, schema] of Object.entries(federated)) {
|
|
4685
|
+
const typeDefs = schema.types ?? {};
|
|
4686
|
+
const types = {};
|
|
4687
|
+
for (const [name, def] of Object.entries(typeDefs)) types[safe(name, NAME)] = Object.entries(def.fields ?? {}).map(([field, shape]) => `${safe(field, NAME)}: ${fieldType(shape, ns)}`).join(", ");
|
|
4688
|
+
const mutations = Object.keys(schema.mutations ?? {}).filter((m) => {
|
|
4689
|
+
const crud = AUTO_CRUD.exec(m);
|
|
4690
|
+
return !(crud?.[2] !== void 0 && crud[2] in typeDefs);
|
|
4691
|
+
}).map((m) => safe(m, NAME));
|
|
4692
|
+
out[safe(ns, NAME)] = mutations.length > 0 ? {
|
|
4693
|
+
types,
|
|
4694
|
+
mutations
|
|
4695
|
+
} : { types };
|
|
4696
|
+
}
|
|
4697
|
+
return out;
|
|
4698
|
+
}
|
|
4699
|
+
/** How to read the overview; rides beside it in the tool result and the instructions. */
|
|
4700
|
+
const OVERVIEW_LEGEND = "Each type is `field: type` pairs. `a|b` is an enum, `?` nullable, `T[]` a list, `→Type` a reference (`→ns/Type` across namespaces). [unindexed] fields need `_allowTableScan` to filter on; [readonly] fields are server-set. Generated get/create/update/delete<Type> mutations are not listed; `mutations` holds the rest. For a type's exact write contract call `schema` with `namespace` and `type`.";
|
|
4701
|
+
/** Past this, the instructions carry only type names; the full overview is one `schema` call away. */
|
|
4702
|
+
const INSTRUCTIONS_OVERVIEW_MAX = 12e3;
|
|
4703
|
+
const CONVENTIONS = [
|
|
4704
|
+
"Substrate holds typed records. Every query and mutate key is qualified: `<namespace>/<Type>` or `<namespace>/<mutation>`.",
|
|
4705
|
+
"- Filters go in `params.where`, never top-level params: `{ \"ns/Type\": { \"params\": { \"where\": { \"status\": { \"eq\": \"open\" } }, \"limit\": 20 }, \"fields\": { \"id\": true } } }`. A refused operator is named with the field and its kind; read it rather than guessing.",
|
|
4706
|
+
"- Read `errors` before `data`. A refused frame is ABSENT from `data`; a present empty list means nothing matched. A read with `_pagination.hasMore` is truncated: page with `nextCursor` (default page 100, max 1000).",
|
|
4707
|
+
"- Lists, counts and joins are exact `query` frames; `recall` (similarity search) needs `query`, `namespace` and `type`, and is for finding records by meaning.",
|
|
4708
|
+
"- In one `mutate` call the single-frame keys commit together or not at all, and an array under one key is one transaction. Send unrelated writes as separate calls.",
|
|
4709
|
+
"- `createdAt`, `updatedAt`, `createdBy` and `updatedBy` are server-set: never send them; read them through `_metadata`.",
|
|
4710
|
+
"- `whoami` says which identity this session reads and writes as.",
|
|
4711
|
+
"- With a shell, `substrate run <script.mts>` runs many reads and writes in one call and returns one small report (GET /substrate/v1/onboard, `instructions.codeMode`)."
|
|
4712
|
+
].join("\n");
|
|
4713
|
+
/** The MCP `instructions` text: the conventions, then this server's types as of session start. */
|
|
4714
|
+
function buildInstructions(overview) {
|
|
4715
|
+
if (overview === null) return `${CONVENTIONS}\n\nThe type list was unavailable at session start; call \`schema\` with no arguments for it.`;
|
|
4716
|
+
const full = JSON.stringify(overview);
|
|
4717
|
+
const body = full.length <= INSTRUCTIONS_OVERVIEW_MAX ? `${OVERVIEW_LEGEND}\n${full}` : `Type names only (the full overview is large; call \`schema\` with no arguments for fields):\n${JSON.stringify(Object.fromEntries(Object.entries(overview).map(([ns, o]) => [ns, Object.keys(o.types)])))}`;
|
|
4718
|
+
return `${CONVENTIONS}\n\nTypes on this server at session start:\n${body}`;
|
|
4719
|
+
}
|
|
4720
|
+
|
|
4644
4721
|
//#endregion
|
|
4645
4722
|
//#region src/mcp/server.ts
|
|
4646
4723
|
/**
|
|
@@ -4796,6 +4873,15 @@ function createSchemaHandler(endpoint) {
|
|
|
4796
4873
|
const ns = args?.namespace;
|
|
4797
4874
|
const type = args?.type;
|
|
4798
4875
|
const shape = args?.shape;
|
|
4876
|
+
if ((ns === void 0 || ns === "") && args?.full !== true) try {
|
|
4877
|
+
const federated = await getSubstrate(endpoint, "/substrate/v1/federated-schema");
|
|
4878
|
+
return okResult({
|
|
4879
|
+
legend: OVERVIEW_LEGEND,
|
|
4880
|
+
overview: schemaOverview(federated)
|
|
4881
|
+
});
|
|
4882
|
+
} catch (err) {
|
|
4883
|
+
return errorResult(toMessage(err));
|
|
4884
|
+
}
|
|
4799
4885
|
let path = "/substrate/v1/federated-schema";
|
|
4800
4886
|
if (ns !== void 0 && ns !== "") {
|
|
4801
4887
|
if (type !== void 0 && type !== "") {
|
|
@@ -4903,16 +4989,17 @@ function createMcpServer(endpoint, options = {}) {
|
|
|
4903
4989
|
const server = new McpServer({
|
|
4904
4990
|
name: "substrate",
|
|
4905
4991
|
version: MCP_SERVER_VERSION
|
|
4906
|
-
});
|
|
4992
|
+
}, options.instructions !== void 0 ? { instructions: options.instructions } : void 0);
|
|
4907
4993
|
const TOOLS = [
|
|
4908
4994
|
{
|
|
4909
4995
|
name: "schema",
|
|
4910
4996
|
config: {
|
|
4911
|
-
description: "Fetch the Substrate schema. No args →
|
|
4997
|
+
description: "Fetch the Substrate schema. No args → a compact overview: every namespace's types, one line of `field: type` each, enough to write a query. `full: true` → the whole federated schema (large; rarely needed). SCOPED + CHEAP: pass `namespace` for one namespace, or `namespace` + `type` for a single type's write contract ({ fields, mutations }). Mine the scoped shape BEFORE a create/update so you send only declared fields — autoCRUD rejects undeclared fields with `unknown_field`. CHEAPER STILL: add `shape: \"fields\"` when you only need the type's field names and kinds — for an autoCRUD type the mutations restate the field block five times over, so the default answer costs about 6x the question.",
|
|
4912
4998
|
inputSchema: {
|
|
4913
4999
|
namespace: z.string().min(1).optional().describe("Scope to one namespace, e.g. \"notes\". Omit for the full federated schema."),
|
|
4914
5000
|
type: z.string().min(1).optional().describe("Scope to one type within `namespace`, e.g. \"Todo\" — returns its field defs + create/update param shapes. Requires `namespace`."),
|
|
4915
|
-
shape: z.enum(["full", "fields"]).optional().describe("Projection of the type contract; requires `namespace` + `type`. \"full\" (default) inlines every mutation's params and returns. \"fields\" returns the field defs plus `mutationNames`, about a sixth the size — use it to answer \"what fields does this type have\".")
|
|
5001
|
+
shape: z.enum(["full", "fields"]).optional().describe("Projection of the type contract; requires `namespace` + `type`. \"full\" (default) inlines every mutation's params and returns. \"fields\" returns the field defs plus `mutationNames`, about a sixth the size — use it to answer \"what fields does this type have\"."),
|
|
5002
|
+
full: z.boolean().optional().describe("With no `namespace`: true returns the whole federated schema instead of the overview.")
|
|
4916
5003
|
}
|
|
4917
5004
|
},
|
|
4918
5005
|
handler: createSchemaHandler(endpoint)
|
|
@@ -5064,6 +5151,12 @@ function resolveToolAllowlist(raw, known) {
|
|
|
5064
5151
|
for (const n of names) if (!known.includes(n)) throw new Error(`SUBSTRATE_MCP_TOOLS: unknown tool '${n}'. Valid names: ${[...known].sort().join(", ")}`);
|
|
5065
5152
|
return new Set(names);
|
|
5066
5153
|
}
|
|
5154
|
+
/** The overview for the session's instructions, or null when the server is slow or down. */
|
|
5155
|
+
async function fetchOverview(endpoint) {
|
|
5156
|
+
const timeout = new Promise((resolve) => setTimeout(() => resolve(null), 5e3).unref());
|
|
5157
|
+
const fetched = getSubstrate(endpoint, "/substrate/v1/federated-schema").then(schemaOverview, () => null);
|
|
5158
|
+
return Promise.race([fetched, timeout]);
|
|
5159
|
+
}
|
|
5067
5160
|
/**
|
|
5068
5161
|
* Boot the stdio MCP server. Resolves once the transport is connected; the
|
|
5069
5162
|
* process keeps running until the host (Claude Code, etc.) closes stdio.
|
|
@@ -5071,7 +5164,10 @@ function resolveToolAllowlist(raw, known) {
|
|
|
5071
5164
|
async function runMcp() {
|
|
5072
5165
|
const endpoint = process.env.SUBSTRATE_URL ?? "http://localhost:3100";
|
|
5073
5166
|
configureAgentAuth(process.env, endpoint);
|
|
5074
|
-
const server = createMcpServer(endpoint,
|
|
5167
|
+
const server = createMcpServer(endpoint, {
|
|
5168
|
+
...process.env.SUBSTRATE_MCP_TOOLS === void 0 ? {} : { tools: process.env.SUBSTRATE_MCP_TOOLS },
|
|
5169
|
+
instructions: buildInstructions(await fetchOverview(endpoint))
|
|
5170
|
+
});
|
|
5075
5171
|
const transport = new StdioServerTransport();
|
|
5076
5172
|
await server.connect(transport);
|
|
5077
5173
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mnmal-ai/substrate",
|
|
3
|
-
"version": "1.0.0-alpha.
|
|
3
|
+
"version": "1.0.0-alpha.9",
|
|
4
4
|
"description": "Substrate — an agent-first typed-data server: the substrate and substrate-admin binaries, the built-in plugins and the SDK subpaths.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"repository": {
|
|
@@ -36,24 +36,24 @@
|
|
|
36
36
|
},
|
|
37
37
|
"devDependencies": {
|
|
38
38
|
"@inquirer/prompts": "^8.7.2",
|
|
39
|
-
"@mnmal-ai/substrate-bootstrap": "1.0.0-alpha.
|
|
40
|
-
"@mnmal-ai/substrate-client": "1.0.0-alpha.
|
|
41
|
-
"@mnmal-ai/substrate-compose": "1.0.0-alpha.
|
|
42
|
-
"@mnmal-ai/substrate-config": "1.0.0-alpha.
|
|
43
|
-
"@mnmal-ai/substrate-core": "1.0.0-alpha.
|
|
44
|
-
"@mnmal-ai/substrate-dataset-memory": "1.0.0-alpha.
|
|
45
|
-
"@mnmal-ai/substrate-dataset-pg": "1.0.0-alpha.
|
|
46
|
-
"@mnmal-ai/substrate-embedder": "1.0.0-alpha.
|
|
47
|
-
"@mnmal-ai/substrate-filter": "1.0.0-alpha.
|
|
48
|
-
"@mnmal-ai/substrate-pack": "1.0.0-alpha.
|
|
49
|
-
"@mnmal-ai/substrate-policy": "1.0.0-alpha.
|
|
50
|
-
"@mnmal-ai/substrate-proposals": "1.0.0-alpha.
|
|
51
|
-
"@mnmal-ai/substrate-recall": "1.0.0-alpha.
|
|
52
|
-
"@mnmal-ai/substrate-sdk": "1.0.0-alpha.
|
|
53
|
-
"@mnmal-ai/substrate-sentinel-client": "1.0.0-alpha.
|
|
54
|
-
"@mnmal-ai/substrate-server-fastify": "1.0.0-alpha.
|
|
55
|
-
"@mnmal-ai/substrate-server-memory": "1.0.0-alpha.
|
|
56
|
-
"@mnmal-ai/substrate-types": "1.0.0-alpha.
|
|
39
|
+
"@mnmal-ai/substrate-bootstrap": "1.0.0-alpha.9",
|
|
40
|
+
"@mnmal-ai/substrate-client": "1.0.0-alpha.9",
|
|
41
|
+
"@mnmal-ai/substrate-compose": "1.0.0-alpha.9",
|
|
42
|
+
"@mnmal-ai/substrate-config": "1.0.0-alpha.9",
|
|
43
|
+
"@mnmal-ai/substrate-core": "1.0.0-alpha.9",
|
|
44
|
+
"@mnmal-ai/substrate-dataset-memory": "1.0.0-alpha.9",
|
|
45
|
+
"@mnmal-ai/substrate-dataset-pg": "1.0.0-alpha.9",
|
|
46
|
+
"@mnmal-ai/substrate-embedder": "1.0.0-alpha.9",
|
|
47
|
+
"@mnmal-ai/substrate-filter": "1.0.0-alpha.9",
|
|
48
|
+
"@mnmal-ai/substrate-pack": "1.0.0-alpha.9",
|
|
49
|
+
"@mnmal-ai/substrate-policy": "1.0.0-alpha.9",
|
|
50
|
+
"@mnmal-ai/substrate-proposals": "1.0.0-alpha.9",
|
|
51
|
+
"@mnmal-ai/substrate-recall": "1.0.0-alpha.9",
|
|
52
|
+
"@mnmal-ai/substrate-sdk": "1.0.0-alpha.9",
|
|
53
|
+
"@mnmal-ai/substrate-sentinel-client": "1.0.0-alpha.9",
|
|
54
|
+
"@mnmal-ai/substrate-server-fastify": "1.0.0-alpha.9",
|
|
55
|
+
"@mnmal-ai/substrate-server-memory": "1.0.0-alpha.9",
|
|
56
|
+
"@mnmal-ai/substrate-types": "1.0.0-alpha.9",
|
|
57
57
|
"@types/node": "^26.6.4",
|
|
58
58
|
"@types/semver": "^7.5.0",
|
|
59
59
|
"@types/ws": "^8.18.1",
|