@klum-db/lobby 0.2.0-pre.30 → 0.2.0-pre.32

package/dist/index.d.cts

@@ -1,7 +1,7 @@
 import { Vault, SealingKeyProvider, PublicEnvelope, Noydb } from '@noy-db/hub';
 export { CustodyApi, DeedMarker, GrantCustodianOptions, LiberateOptions, LiberateResult, createDeedOwner, isDeedVault, liberateVault, loadDeedMarker } from '@noy-db/hub';
-import { a as VaultTemplate, S as StateManagementVault, V as VaultGroup, b as SkippedVault, M as MergeCompartmentOptions, c as MergeReport, d as SurfaceDirection, e as SurfaceConflictPolicy, f as SurfaceRow, g as VaultGroupOptions } from './vault-group-BXjO5kHB.cjs';
-export { C as CapturedBlueprint, h as CrossVaultAggregation, i as CrossVaultDerivationContext, j as CrossVaultDerivationSpec, k as CrossVaultGroupedAggregation, G as CrossVaultGroupedRow, l as CrossVaultLiveAggregation, m as CrossVaultLiveQuery, D as DecryptedMergeOptions, n as DeploymentEvent, F as FanoutQueryOptions, o as FanoutResult, p as FieldAuthorityInputs, q as FieldAuthorityPolicy, r as FieldAuthorityPolicyMissingError, s as FieldAuthorityRule, t as FieldLevelDeferredError, L as LiveQueryOptions, u as MergeConflict, v as MergeStrategy, w as MigrationStatusRow, R as RefreshInsightsResult, x as SchemaManifestRow, y as SchemaRolloutResult, z as ShardedCollection, A as ShardedGroupedQuery, B as ShardedQuery, E as ShardingConfig, H as SurfaceStatus, I as VaultRegistryRow, J as mergeCompartment, K as mergeDecryptedRecords, N as resolveFieldAuthority, O as resolveRecordByFieldAuthority } from './vault-group-BXjO5kHB.cjs';
+import { a as VaultTemplate, S as StateManagementVault, V as VaultGroup, b as SkippedVault, M as MergeCompartmentOptions, c as MergeReport, d as SurfaceDirection, e as SurfaceConflictPolicy, f as SurfaceRow, g as VaultGroupOptions } from './vault-group-BchRCDm5.cjs';
+export { C as CapturedBlueprint, h as CrossVaultAggregation, i as CrossVaultDerivationContext, j as CrossVaultDerivationSpec, k as CrossVaultGroupedAggregation, G as CrossVaultGroupedRow, l as CrossVaultLiveAggregation, m as CrossVaultLiveQuery, D as DecryptedMergeOptions, n as DeploymentEvent, F as FanoutQueryOptions, o as FanoutResult, p as FederatedRetrieveHit, q as FederatedRetrieveOptions, r as FederatedRetrieveResult, s as FederationMeta, t as FieldAuthorityInputs, u as FieldAuthorityPolicy, v as FieldAuthorityPolicyMissingError, w as FieldAuthorityRule, x as FieldLevelDeferredError, y as GroupMeta, L as LiveQueryOptions, z as MergeConflict, A as MergeStrategy, B as MigrationStatusRow, R as RefreshInsightsResult, E as SchemaManifestRow, H as SchemaRolloutResult, I as ShardedCollection, J as ShardedGroupedQuery, K as ShardedQuery, N as ShardingConfig, O as SurfaceStatus, P as VaultRegistryRow, Q as mergeCompartment, T as mergeDecryptedRecords, U as resolveFieldAuthority, W as resolveRecordByFieldAuthority } from './vault-group-BchRCDm5.cjs';
 import { AsXlsxSheetOptions } from '@noy-db/as-xlsx';
 export { MultiVaultDenormColumn, MultiVaultXlsxEntry, MultiVaultXlsxOptions } from '@noy-db/as-xlsx';
 import { ExtractionPreview, WriteNoydbBundleOptions } from '@noy-db/hub/bundle';

package/dist/index.d.ts

@@ -1,7 +1,7 @@
 import { Vault, SealingKeyProvider, PublicEnvelope, Noydb } from '@noy-db/hub';
 export { CustodyApi, DeedMarker, GrantCustodianOptions, LiberateOptions, LiberateResult, createDeedOwner, isDeedVault, liberateVault, loadDeedMarker } from '@noy-db/hub';
-import { a as VaultTemplate, S as StateManagementVault, V as VaultGroup, b as SkippedVault, M as MergeCompartmentOptions, c as MergeReport, d as SurfaceDirection, e as SurfaceConflictPolicy, f as SurfaceRow, g as VaultGroupOptions } from './vault-group-BXjO5kHB.js';
-export { C as CapturedBlueprint, h as CrossVaultAggregation, i as CrossVaultDerivationContext, j as CrossVaultDerivationSpec, k as CrossVaultGroupedAggregation, G as CrossVaultGroupedRow, l as CrossVaultLiveAggregation, m as CrossVaultLiveQuery, D as DecryptedMergeOptions, n as DeploymentEvent, F as FanoutQueryOptions, o as FanoutResult, p as FieldAuthorityInputs, q as FieldAuthorityPolicy, r as FieldAuthorityPolicyMissingError, s as FieldAuthorityRule, t as FieldLevelDeferredError, L as LiveQueryOptions, u as MergeConflict, v as MergeStrategy, w as MigrationStatusRow, R as RefreshInsightsResult, x as SchemaManifestRow, y as SchemaRolloutResult, z as ShardedCollection, A as ShardedGroupedQuery, B as ShardedQuery, E as ShardingConfig, H as SurfaceStatus, I as VaultRegistryRow, J as mergeCompartment, K as mergeDecryptedRecords, N as resolveFieldAuthority, O as resolveRecordByFieldAuthority } from './vault-group-BXjO5kHB.js';
+import { a as VaultTemplate, S as StateManagementVault, V as VaultGroup, b as SkippedVault, M as MergeCompartmentOptions, c as MergeReport, d as SurfaceDirection, e as SurfaceConflictPolicy, f as SurfaceRow, g as VaultGroupOptions } from './vault-group-BchRCDm5.js';
+export { C as CapturedBlueprint, h as CrossVaultAggregation, i as CrossVaultDerivationContext, j as CrossVaultDerivationSpec, k as CrossVaultGroupedAggregation, G as CrossVaultGroupedRow, l as CrossVaultLiveAggregation, m as CrossVaultLiveQuery, D as DecryptedMergeOptions, n as DeploymentEvent, F as FanoutQueryOptions, o as FanoutResult, p as FederatedRetrieveHit, q as FederatedRetrieveOptions, r as FederatedRetrieveResult, s as FederationMeta, t as FieldAuthorityInputs, u as FieldAuthorityPolicy, v as FieldAuthorityPolicyMissingError, w as FieldAuthorityRule, x as FieldLevelDeferredError, y as GroupMeta, L as LiveQueryOptions, z as MergeConflict, A as MergeStrategy, B as MigrationStatusRow, R as RefreshInsightsResult, E as SchemaManifestRow, H as SchemaRolloutResult, I as ShardedCollection, J as ShardedGroupedQuery, K as ShardedQuery, N as ShardingConfig, O as SurfaceStatus, P as VaultRegistryRow, Q as mergeCompartment, T as mergeDecryptedRecords, U as resolveFieldAuthority, W as resolveRecordByFieldAuthority } from './vault-group-BchRCDm5.js';
 import { AsXlsxSheetOptions } from '@noy-db/as-xlsx';
 export { MultiVaultDenormColumn, MultiVaultXlsxEntry, MultiVaultXlsxOptions } from '@noy-db/as-xlsx';
 import { ExtractionPreview, WriteNoydbBundleOptions } from '@noy-db/hub/bundle';

package/dist/index.js

@@ -1254,6 +1254,43 @@ var init_insight_auto_push = __esm({
   }
 });
 
+// src/federation/partial-reduce.ts
+function canPartialReduce(spec) {
+  return Object.values(spec).every((r) => typeof r.merge === "function");
+}
+function reduceToPartial(records, spec) {
+  const out = {};
+  for (const [key, reducer] of Object.entries(spec)) {
+    const r = reducer;
+    let state = r.init();
+    for (const rec of records) state = r.step(state, rec);
+    out[key] = state;
+  }
+  return out;
+}
+function mergePartials(spec, partials) {
+  const out = {};
+  for (const [key, reducer] of Object.entries(spec)) {
+    const r = reducer;
+    let acc = r.init();
+    for (const p of partials) acc = r.merge(acc, p[key]);
+    out[key] = acc;
+  }
+  return out;
+}
+function finalizePartial(spec, merged) {
+  const out = {};
+  for (const [key, reducer] of Object.entries(spec)) {
+    out[key] = reducer.finalize(merged[key]);
+  }
+  return out;
+}
+var init_partial_reduce = __esm({
+  "src/federation/partial-reduce.ts"() {
+    "use strict";
+  }
+});
+
 // src/federation/aggregate-across.ts
 import { reduceRecords } from "@noy-db/hub/kernel";
 import { groupAndReduce } from "@noy-db/hub/kernel";
@@ -1262,6 +1299,7 @@ var init_aggregate_across = __esm({
   "src/federation/aggregate-across.ts"() {
     "use strict";
     init_cross_vault_live();
+    init_partial_reduce();
     CrossVaultAggregation = class {
       constructor(src, spec, bind) {
         this.src = src;
@@ -1272,6 +1310,10 @@ var init_aggregate_across = __esm({
       spec;
       bind;
       async run(options = {}) {
+        if (this.src.fanoutReduce && canPartialReduce(this.spec)) {
+          const { partials, skippedVaults: skippedVaults2 } = await this.src.fanoutReduce(this.spec, options);
+          return { result: finalizePartial(this.spec, mergePartials(this.spec, partials)), skippedVaults: skippedVaults2 };
+        }
         const { records, skippedVaults } = await this.src.fanoutRecords(options);
         return { result: reduceRecords(records, this.spec), skippedVaults };
       }
@@ -1360,6 +1402,73 @@ var init_aggregate_across = __esm({
   }
 });
 
+// src/federation/retrieve-across.ts
+import { fuseRetrieval } from "@noy-db/hub/kernel";
+async function retrieveAcross(group, collectionName, query, opts = {}) {
+  const { eligible, skipped } = await group.resolveEligible({
+    ...opts.minVersion !== void 0 ? { minVersion: opts.minVersion } : {},
+    ...opts.failFast !== void 0 ? { failFast: opts.failFast } : {}
+  });
+  const across = await group.db.queryAcross(
+    eligible.map((r) => r.vaultId),
+    async (vault) => {
+      group.template.configure(vault);
+      const coll = vault.collection(collectionName);
+      let within;
+      if (opts.where?.length) {
+        let q = coll.query();
+        for (const [f, op, v] of opts.where) q = q.where(f, op, v);
+        within = q;
+      }
+      return coll.retrieve(query, {
+        ...opts.mode ? { mode: opts.mode } : {},
+        ...opts.limit !== void 0 ? { limit: opts.limit } : {},
+        ...opts.minScore !== void 0 ? { minScore: opts.minScore } : {},
+        ...opts.fields ? { fields: opts.fields } : {},
+        ...opts.match ? { match: opts.match } : {},
+        ...opts.prefix ? { prefix: opts.prefix } : {},
+        ...opts.snippetWindow !== void 0 ? { snippetWindow: opts.snippetWindow } : {},
+        ...opts.includeRecord ? { includeRecord: true } : {},
+        ...within ? { within } : {}
+      });
+    },
+    { concurrency: opts.concurrency ?? 1, create: false }
+  );
+  const skippedVaults = [...skipped];
+  const perVault = [];
+  for (const r of across) {
+    if (r.error) {
+      if (opts.failFast) throw r.error;
+      skippedVaults.push({ vaultId: r.vault, reason: classifyShardSkip(r.error), error: r.error });
+    } else {
+      perVault.push({ vault: r.vault, hits: r.result });
+    }
+  }
+  const lists = perVault.map((pv) => {
+    if (pv.vault.includes(SEP)) {
+      throw new Error(`retrieveAcross: vault id "${pv.vault}" contains the reserved NUL separator`);
+    }
+    return pv.hits.map((h) => ({ ...h, id: pv.vault + SEP + h.id }));
+  });
+  const fused = fuseRetrieval(lists, {
+    ...opts.limit !== void 0 ? { limit: opts.limit } : {},
+    ...opts.rrfK !== void 0 ? { k: opts.rrfK } : {}
+  });
+  const hits = fused.map((h) => {
+    const i = h.id.indexOf(SEP);
+    return { ...h, vault: h.id.slice(0, i), id: h.id.slice(i + 1) };
+  });
+  return { hits, skippedVaults };
+}
+var SEP;
+var init_retrieve_across = __esm({
+  "src/federation/retrieve-across.ts"() {
+    "use strict";
+    init_classify_skip();
+    SEP = "\0";
+  }
+});
+
 // src/federation/vault-group.ts
 var vault_group_exports = {};
 __export(vault_group_exports, {
@@ -1398,16 +1507,19 @@ var init_vault_group = __esm({
     init_cross_vault_live();
     init_insight_auto_push();
     init_aggregate_across();
+    init_retrieve_across();
+    init_partial_reduce();
     SHARD_SEPARATOR = "--";
     SAFE_PARTITION_KEY = /^[A-Za-z0-9._-]+$/;
     VaultGroup = class {
-      constructor(db, name, registry, sharding, template, cutoverOnOpen = false) {
+      constructor(db, name, registry, sharding, template, cutoverOnOpen = false, meta) {
         this.db = db;
         this.name = name;
         this.registry = registry;
         this.sharding = sharding;
         this.template = template;
         this.cutoverOnOpen = cutoverOnOpen;
+        this.meta = meta;
         if (name.includes(SHARD_SEPARATOR)) {
           throw new ValidationError(
             `VaultGroup name "${name}" must not contain "--" (reserved shard vault-id separator).`
@@ -1420,6 +1532,7 @@ var init_vault_group = __esm({
       sharding;
       template;
       cutoverOnOpen;
+      meta;
       /** @internal — set when the group is managed (no explicit registry). */
       stateVault;
       /** @internal */
@@ -1453,6 +1566,30 @@ var init_vault_group = __esm({
         const rows = this.registry.query().toArray();
         return rows.filter((r) => r.group === this.name);
       }
+      /**
+       * Federation-level descriptive metadata (#27): this group's `meta` plus each
+       * member vault's `vaultMeta` (read via `getMeta()`). Best-effort per shard — a
+       * shard that fails to open yields `meta: undefined` for that entry, never
+       * throwing. Member meta is transient per-open (noy-db sets it at `openVault`,
+       * does not persist it); this surfaces whatever each vault was opened with.
+       */
+      async federationMeta() {
+        const rows = await this.allRows();
+        const vaults = await Promise.all(
+          rows.map(async (row) => {
+            let meta;
+            try {
+              const vault = await this.db.openVault(row.vaultId);
+              this.template.configure(vault);
+              meta = vault.getMeta();
+            } catch {
+              meta = void 0;
+            }
+            return { vaultId: row.vaultId, partitionKey: row.partitionKey, meta };
+          })
+        );
+        return { meta: this.meta, vaults };
+      }
       /**
        * Open an existing shard and apply the template. When `cutoverOnOpen` is set
        * (#271) and the shard's registry version is behind the template, its cutover
@@ -1536,22 +1673,33 @@ var init_vault_group = __esm({
       collection(collectionName) {
         return new ShardedCollection(this, collectionName);
       }
-      /** @internal — eligible (openable-candidate) rows + drift/divergence skips. */
+      /** @internal — eligible (openable-candidate) rows + drift/divergence/unreachable skips. */
       async resolveEligible(options = {}) {
         const rows = await this.allRows();
+        const candidates = options.only ? rows.filter((r) => options.only.includes(r.partitionKey)) : rows;
         const skipped = [];
         const versionOk = [];
-        for (const row of rows) {
+        for (const row of candidates) {
           if (options.minVersion !== void 0 && row.schemaVersion < options.minVersion) {
             skipped.push({ vaultId: row.vaultId, reason: "schema-drift" });
           } else versionOk.push(row);
         }
-        const provisioned = await Promise.all(versionOk.map((r) => this.db._shardVaultProvisioned(r.vaultId)));
+        const probes = await Promise.all(
+          versionOk.map(async (row) => {
+            try {
+              return { row, provisioned: await this.db._shardVaultProvisioned(row.vaultId) };
+            } catch (err) {
+              if (options.failFast) throw err;
+              return { row, error: err };
+            }
+          })
+        );
         const eligible = [];
-        versionOk.forEach((row, i) => {
-          if (provisioned[i]) eligible.push(row);
-          else skipped.push({ vaultId: row.vaultId, reason: "error", error: new ShardProvisioningError(row.vaultId, row.partitionKey) });
-        });
+        for (const p of probes) {
+          if ("error" in p) skipped.push({ vaultId: p.row.vaultId, reason: "error", error: p.error });
+          else if (p.provisioned) eligible.push(p.row);
+          else skipped.push({ vaultId: p.row.vaultId, reason: "error", error: new ShardProvisioningError(p.row.vaultId, p.row.partitionKey) });
+        }
         return { eligible, skipped };
       }
       /** @internal — registered push-model cross-vault derivations (#271 Insight Vault). */
@@ -1611,12 +1759,19 @@ var init_vault_group = __esm({
        * Insight Vault keyed by partition key. Shards behind `minVersion`,
        * unprovisioned, or whose read errors are reported in `skippedVaults` and
        * are not written (a stale summary is never left behind for a failed shard).
+       * A shard whose backend is unreachable (provisioning probe or read throws) is
+       * **skipped** (`reason: 'error'`) and the pass continues for the others — its
+       * prior summary is left intact. Pass `failFast: true` for the legacy
+       * all-or-nothing throw. Reconcile a lagging shard later with
+       * `refreshInsights({ only: [pk] })` or `refreshDerivation(pk)`.
        */
       async refreshInsights(options = {}) {
         if (this.crossVaultDerivations.length === 0) return { written: 0, skippedVaults: [] };
-        const { eligible, skipped } = await this.resolveEligible(
-          options.minVersion !== void 0 ? { minVersion: options.minVersion } : {}
-        );
+        const { eligible, skipped } = await this.resolveEligible({
+          ...options.minVersion !== void 0 ? { minVersion: options.minVersion } : {},
+          ...options.only !== void 0 ? { only: options.only } : {},
+          ...options.failFast !== void 0 ? { failFast: options.failFast } : {}
+        });
         let written = 0;
         for (const spec of this.crossVaultDerivations) {
           const results = await this.db.queryAcross(
@@ -1633,6 +1788,7 @@ var init_vault_group = __esm({
             const row = eligible[i];
             const res = results[i];
             if (!res || res.result === void 0) {
+              if (options.failFast && res?.error) throw res.error;
               skipped.push({ vaultId: row.vaultId, reason: "error", ...res?.error ? { error: res.error } : {} });
               continue;
             }
@@ -1648,6 +1804,14 @@ var init_vault_group = __esm({
         }
         return { written, skippedVaults: skipped };
       }
+      /**
+       * Reconcile one shard's Insight summaries after its backend was unreachable.
+       * Equivalent to `refreshInsights({ only: [partitionKey] })` — runs every
+       * registered derivation (autoPush or not) for just this shard.
+       */
+      async refreshDerivation(partitionKey) {
+        return this.refreshInsights({ only: [partitionKey] });
+      }
       /** @internal — re-derive + push every autoPush derivation's summary for one shard. */
       async _recomputeShardInsights(partitionKey) {
         const row = await this.registry.get(this.registryId(partitionKey));
@@ -1787,6 +1951,17 @@ var init_vault_group = __esm({
       query() {
         return new ShardedQuery(this.group, this.collectionName, []);
       }
+      /**
+       * Cross-vault federated retrieval (#26): scatter-gather across all eligible
+       * shards — each shard runs its own trusted-tier `retrieve()`, then results are
+       * RRF-fused by rank only (no cross-vault statistics cross the DEK boundary).
+       * Every hit carries its originating `vault`. An unreachable, schema-drifted, or
+       * un-indexed shard is skipped (`skippedVaults`); pass `failFast` to re-throw
+       * the first per-shard error instead.
+       */
+      async retrieve(query, opts = {}) {
+        return retrieveAcross(this.group, this.collectionName, query, opts);
+      }
     };
     ShardedQuery = class _ShardedQuery {
       constructor(group, collectionName, clauses, coPartitionedLegs = [], broadcastLegs = []) {
@@ -1883,6 +2058,34 @@ var init_vault_group = __esm({
         }
         return { records: results, skippedVaults: skipped };
       }
+      /**
+       * @internal — distributed partial-reduce (#8). Fan out across eligible shards,
+       * but fold each shard's where-filtered records to a partial reducer STATE
+       * inside the per-shard callback (never returning the rows). Only the scalar
+       * `.aggregate()` path calls this, and that path rejects join legs upstream.
+       */
+      async fanoutReduce(spec, options = {}) {
+        const { eligible, skipped } = await this.group.resolveEligible(options);
+        const across = await this.group.db.queryAcross(
+          eligible.map((r) => r.vaultId),
+          async (vault) => {
+            this.group.template.configure(vault);
+            const coll = vault.collection(this.collectionName);
+            await coll.list();
+            let q = coll.query();
+            for (const c of this.clauses) q = q.where(c.field, c.op, c.value);
+            const rows = q.toArray();
+            return reduceToPartial(rows, spec);
+          },
+          { concurrency: options.concurrency ?? 1, create: false }
+        );
+        const partials = [];
+        for (const r of across) {
+          if (r.error) skipped.push({ vaultId: r.vault, reason: classifyShardSkip(r.error), error: r.error });
+          else partials.push(r.result);
+        }
+        return { partials, skippedVaults: skipped };
+      }
       /** Fan out across eligible shards, merge, then apply any broadcast dimension legs. */
       async toArray(options = {}) {
         const { records, skippedVaults } = await this.fanoutRecords(options);
@@ -2154,7 +2357,7 @@ var Lobby = class {
     const { StateManagementVault: StateManagementVault2 } = await Promise.resolve().then(() => (init_state_vault(), state_vault_exports));
     const stateVault = opts.registry ? void 0 : await StateManagementVault2.open(db);
     const registry = opts.registry ?? stateVault.registry;
-    const group = new VaultGroup2(db, name, registry, opts.sharding, template, opts.cutoverOnOpen ?? false);
+    const group = new VaultGroup2(db, name, registry, opts.sharding, template, opts.cutoverOnOpen ?? false, opts.meta);
     if (stateVault) {
       group._attachStateVault(stateVault);
       await stateVault.recordManifest(opts.sharding.vaultTemplate, template);