@gmickel/gno 1.27.1 → 1.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (195) hide show
  1. package/README.md +46 -20
  2. package/assets/skill/SKILL.md +59 -1
  3. package/browser-extension/artifacts/gno-browser-clipper-v1.29.0.zip +0 -0
  4. package/browser-extension/artifacts/gno-browser-clipper-v1.29.0.zip.sha256 +1 -0
  5. package/browser-extension/dist/chunk-b2zm0jjd.js +50 -0
  6. package/browser-extension/dist/manifest.json +1 -1
  7. package/browser-extension/dist/preview.html +1 -1
  8. package/browser-extension/dist/service-worker.js +6 -6
  9. package/package.json +2 -2
  10. package/spec/cli.md +82 -5
  11. package/spec/db/schema.sql +31 -0
  12. package/spec/mcp.md +71 -0
  13. package/spec/output-schemas/ask.schema.json +156 -1
  14. package/spec/output-schemas/browser-clip-preview.schema.json +30 -0
  15. package/spec/output-schemas/collection-egress-check.schema.json +91 -0
  16. package/spec/output-schemas/collection-egress-policy-set.schema.json +56 -0
  17. package/spec/output-schemas/collection-egress-policy.schema.json +41 -0
  18. package/spec/output-schemas/context-capsule-v1.schema.json +183 -3
  19. package/spec/output-schemas/context-capsule-verification.schema.json +1 -1
  20. package/spec/output-schemas/egress-audit-management.schema.json +88 -0
  21. package/spec/output-schemas/get.schema.json +94 -0
  22. package/spec/output-schemas/mcp-http-error.schema.json +113 -2
  23. package/spec/output-schemas/mcp-job-status.schema.json +28 -0
  24. package/spec/output-schemas/multi-get.schema.json +128 -0
  25. package/spec/output-schemas/publish-artifact.schema.json +32 -0
  26. package/spec/output-schemas/record-import.schema.json +223 -0
  27. package/spec/output-schemas/retrieval-trace-common.schema.json +31 -0
  28. package/spec/output-schemas/retrieval-trace-export.schema.json +3 -0
  29. package/spec/output-schemas/retrieval-trace-qrels.schema.json +3 -0
  30. package/spec/output-schemas/retrieval-trace-summary.schema.json +4 -0
  31. package/spec/output-schemas/search-result.schema.json +94 -0
  32. package/spec/output-schemas/search-results.schema.json +155 -0
  33. package/spec/project-profile.schema.json +66 -0
  34. package/src/app/context-format.ts +1 -1
  35. package/src/app/context-runtime-contract.ts +16 -7
  36. package/src/cli/commands/ask.ts +7 -0
  37. package/src/cli/commands/collection/index.ts +5 -0
  38. package/src/cli/commands/collection/policy.ts +126 -0
  39. package/src/cli/commands/context-build.ts +5 -0
  40. package/src/cli/commands/daemon.ts +45 -3
  41. package/src/cli/commands/egress-audit.ts +68 -0
  42. package/src/cli/commands/embed.ts +2 -0
  43. package/src/cli/commands/get.ts +12 -2
  44. package/src/cli/commands/index-cmd.ts +13 -0
  45. package/src/cli/commands/multi-get.ts +9 -2
  46. package/src/cli/commands/query.ts +12 -0
  47. package/src/cli/commands/replay.ts +13 -3
  48. package/src/cli/commands/shared.ts +28 -0
  49. package/src/cli/commands/update.ts +5 -0
  50. package/src/cli/commands/vsearch.ts +3 -1
  51. package/src/cli/program.ts +127 -0
  52. package/src/config/index.ts +9 -0
  53. package/src/config/project-profile.ts +2 -0
  54. package/src/config/types.ts +66 -0
  55. package/src/converters/adapters/browser-export/adapter.ts +199 -0
  56. package/src/converters/adapters/browser-export/formats.ts +358 -0
  57. package/src/converters/adapters/email/adapter.ts +429 -0
  58. package/src/converters/adapters/email/html.ts +162 -0
  59. package/src/converters/adapters/email/mime.ts +454 -0
  60. package/src/converters/adapters/email/parameters.ts +77 -0
  61. package/src/converters/adapters/ical/adapter.ts +475 -0
  62. package/src/converters/adapters/ical/recurrence.ts +186 -0
  63. package/src/converters/adapters/jsonl/adapter.ts +238 -0
  64. package/src/converters/adapters/jsonl/config.ts +105 -0
  65. package/src/converters/adapters/shared/html-text.ts +165 -0
  66. package/src/converters/adapters/shared/record-utils.ts +79 -0
  67. package/src/converters/adapters/shared/utf8-lines.ts +141 -0
  68. package/src/converters/adapters/transcript/adapter.ts +295 -0
  69. package/src/converters/adapters/transcript/json.ts +184 -0
  70. package/src/converters/adapters/transcript/model.ts +171 -0
  71. package/src/converters/adapters/transcript/text.ts +58 -0
  72. package/src/converters/adapters/transcript/timed.ts +152 -0
  73. package/src/converters/index.ts +11 -1
  74. package/src/converters/mime.ts +8 -0
  75. package/src/converters/pipeline.ts +15 -1
  76. package/src/converters/registry.ts +37 -1
  77. package/src/converters/types.ts +136 -0
  78. package/src/core/browser-clip.ts +16 -1
  79. package/src/core/collection-egress-policy-projection.ts +28 -0
  80. package/src/core/collection-egress-policy-service.ts +443 -0
  81. package/src/core/collection-egress-policy-validation.ts +242 -0
  82. package/src/core/config-mutation.ts +31 -19
  83. package/src/core/context-capsule-schema.ts +198 -1
  84. package/src/core/context-capsule-validation.ts +4 -4
  85. package/src/core/context-capsule-verification.ts +5 -1
  86. package/src/core/context-capsule.ts +177 -112
  87. package/src/core/context-evidence.ts +93 -15
  88. package/src/core/destination-classifier.ts +402 -0
  89. package/src/core/document-capabilities.ts +12 -0
  90. package/src/core/egress-audit.ts +239 -0
  91. package/src/core/egress-authorization.ts +50 -0
  92. package/src/core/egress-enforcement.ts +264 -0
  93. package/src/core/egress-policy.ts +440 -0
  94. package/src/core/egress-provenance.ts +336 -0
  95. package/src/core/job-manager.ts +37 -0
  96. package/src/core/network-boundary-inventory.ts +261 -0
  97. package/src/core/project-profile-apply-state.ts +5 -0
  98. package/src/core/project-profile.ts +4 -0
  99. package/src/core/record-metadata.ts +49 -0
  100. package/src/core/retrieval-qrels.ts +6 -0
  101. package/src/core/retrieval-replay.ts +18 -4
  102. package/src/core/retrieval-trace-export.ts +23 -4
  103. package/src/core/retrieval-trace-management-helpers.ts +1 -0
  104. package/src/core/retrieval-trace-management-types.ts +3 -0
  105. package/src/core/retrieval-trace-management.ts +9 -1
  106. package/src/core/retrieval-trace-session.ts +94 -0
  107. package/src/core/retrieval-trace.ts +22 -0
  108. package/src/ingestion/record-adapter-canonical.ts +433 -0
  109. package/src/ingestion/record-adapter.ts +448 -0
  110. package/src/ingestion/record-container.ts +704 -0
  111. package/src/ingestion/record-path.ts +20 -0
  112. package/src/ingestion/record-sync.ts +70 -0
  113. package/src/ingestion/sync.ts +243 -36
  114. package/src/ingestion/types.ts +71 -1
  115. package/src/ingestion/walker.ts +31 -11
  116. package/src/llm/errors.ts +10 -0
  117. package/src/llm/http-inference.ts +175 -0
  118. package/src/llm/http-policy.ts +537 -0
  119. package/src/llm/httpEmbedding.ts +47 -28
  120. package/src/llm/httpGeneration.ts +31 -18
  121. package/src/llm/httpRerank.ts +30 -18
  122. package/src/llm/index.ts +1 -0
  123. package/src/llm/nodeLlamaCpp/adapter.ts +55 -20
  124. package/src/llm/pinned-http-connection.ts +177 -0
  125. package/src/mcp/context.ts +53 -1
  126. package/src/mcp/http-egress.ts +220 -0
  127. package/src/mcp/http-security.ts +32 -21
  128. package/src/mcp/http-session.ts +13 -2
  129. package/src/mcp/http-transport.ts +96 -6
  130. package/src/mcp/sync-egress.ts +24 -0
  131. package/src/mcp/tools/add-collection.ts +4 -0
  132. package/src/mcp/tools/ask.ts +1 -1
  133. package/src/mcp/tools/context.ts +9 -2
  134. package/src/mcp/tools/egress.ts +247 -0
  135. package/src/mcp/tools/embed.ts +1 -0
  136. package/src/mcp/tools/get.ts +10 -2
  137. package/src/mcp/tools/index-cmd.ts +7 -0
  138. package/src/mcp/tools/index.ts +98 -0
  139. package/src/mcp/tools/multi-get.ts +9 -2
  140. package/src/mcp/tools/query.ts +10 -0
  141. package/src/mcp/tools/remove-collection.ts +4 -0
  142. package/src/mcp/tools/sync.ts +6 -0
  143. package/src/mcp/tools/trace.ts +7 -1
  144. package/src/mcp/tools/vsearch.ts +1 -0
  145. package/src/mcp/tools/workspace-write.ts +2 -0
  146. package/src/pipeline/egress-lineage.ts +124 -0
  147. package/src/pipeline/filters.ts +1 -1
  148. package/src/pipeline/graph-retrieval.ts +7 -4
  149. package/src/pipeline/hybrid.ts +21 -4
  150. package/src/pipeline/result-context.ts +12 -4
  151. package/src/pipeline/search.ts +23 -4
  152. package/src/pipeline/types.ts +6 -0
  153. package/src/pipeline/vsearch.ts +67 -28
  154. package/src/publish/artifact.ts +31 -3
  155. package/src/publish/export-service.ts +25 -0
  156. package/src/sdk/client.ts +119 -2
  157. package/src/sdk/documents.ts +13 -5
  158. package/src/sdk/embed.ts +1 -0
  159. package/src/sdk/types.ts +33 -0
  160. package/src/serve/browse-tree.ts +4 -2
  161. package/src/serve/clipper-capture.ts +5 -1
  162. package/src/serve/closed-json.ts +61 -0
  163. package/src/serve/config-sync.ts +4 -1
  164. package/src/serve/context.ts +1 -0
  165. package/src/serve/doc-events.ts +110 -39
  166. package/src/serve/public/components/CollectionEgressCheckPanel.tsx +411 -0
  167. package/src/serve/public/components/CollectionEgressPolicyEditor.tsx +78 -0
  168. package/src/serve/public/components/CollectionModelDialog.tsx +93 -9
  169. package/src/serve/public/components/EgressAuditPanel.tsx +278 -0
  170. package/src/serve/public/globals.built.css +1 -1
  171. package/src/serve/public/pages/Collections.tsx +3 -0
  172. package/src/serve/resident-request.ts +80 -3
  173. package/src/serve/resident-runtime.ts +63 -1
  174. package/src/serve/routes/api.ts +237 -65
  175. package/src/serve/routes/clipper.ts +64 -1
  176. package/src/serve/routes/mcp.ts +6 -0
  177. package/src/serve/routes/traces.ts +27 -4
  178. package/src/serve/server.ts +118 -7
  179. package/src/store/migrations/022-record-export-lineage.ts +42 -0
  180. package/src/store/migrations/023-collection-egress-policy.ts +61 -0
  181. package/src/store/migrations/024-egress-derived-lineage.ts +121 -0
  182. package/src/store/migrations/025-collection-egress-policy-revision.ts +40 -0
  183. package/src/store/migrations/index.ts +8 -0
  184. package/src/store/retrieval-trace-codec.ts +15 -0
  185. package/src/store/sqlite/adapter.ts +284 -14
  186. package/src/store/sqlite/change-journal-store.ts +41 -2
  187. package/src/store/sqlite/egress-audit-store.ts +485 -0
  188. package/src/store/sqlite/retrieval-trace-management-store.ts +47 -6
  189. package/src/store/sqlite/retrieval-trace-retention.ts +2 -0
  190. package/src/store/sqlite/retrieval-trace-rows.ts +8 -1
  191. package/src/store/sqlite/retrieval-trace-store.ts +104 -5
  192. package/src/store/types.ts +166 -1
  193. package/browser-extension/artifacts/gno-browser-clipper-v1.27.1.zip +0 -0
  194. package/browser-extension/artifacts/gno-browser-clipper-v1.27.1.zip.sha256 +0 -1
  195. package/browser-extension/dist/chunk-vn5f663b.js +0 -50
@@ -91,6 +91,7 @@ interface CollectionStats {
91
91
  "override" | "preset" | "default"
92
92
  >;
93
93
  pattern?: string;
94
+ egressPolicy?: CollectionModelDetails["egressPolicy"];
94
95
  }
95
96
 
96
97
  interface StatusResponse {
@@ -411,6 +412,7 @@ export default function Collections({ navigate }: PageProps) {
411
412
  models: config?.models,
412
413
  modelSources: config?.modelSources,
413
414
  pattern: config?.pattern,
415
+ egressPolicy: config?.egressPolicy,
414
416
  };
415
417
  });
416
418
 
@@ -722,6 +724,7 @@ export default function Collections({ navigate }: PageProps) {
722
724
  />
723
725
 
724
726
  <CollectionModelDialog
727
+ availableCollections={collections.map(({ name }) => name)}
725
728
  collection={modelDialogCollection}
726
729
  onOpenChange={(open) => {
727
730
  if (!open) {
@@ -18,6 +18,69 @@ function saturatedResponse(): Response {
18
18
  );
19
19
  }
20
20
 
21
+ function policyChangedResponse(): Response {
22
+ return Response.json(
23
+ {
24
+ error: {
25
+ code: "EGRESS_POLICY_CHANGED",
26
+ message: "Collection policy changed; retry",
27
+ },
28
+ },
29
+ { status: 409 }
30
+ );
31
+ }
32
+
33
+ function wrapResidentStream(
34
+ response: Response,
35
+ isCurrent: () => boolean,
36
+ finish: () => void
37
+ ): Response {
38
+ if (!response.body) {
39
+ finish();
40
+ return response;
41
+ }
42
+ const reader = response.body.getReader();
43
+ let finished = false;
44
+ const finishOnce = (): void => {
45
+ if (finished) return;
46
+ finished = true;
47
+ finish();
48
+ };
49
+ const body = new ReadableStream<Uint8Array>({
50
+ async pull(controller) {
51
+ try {
52
+ if (!isCurrent()) {
53
+ await reader.cancel("EGRESS_POLICY_CHANGED");
54
+ controller.close();
55
+ finishOnce();
56
+ return;
57
+ }
58
+ const result = await reader.read();
59
+ if (!isCurrent()) {
60
+ await reader.cancel("EGRESS_POLICY_CHANGED");
61
+ controller.close();
62
+ finishOnce();
63
+ return;
64
+ }
65
+ if (result.done) {
66
+ controller.close();
67
+ finishOnce();
68
+ return;
69
+ }
70
+ controller.enqueue(result.value);
71
+ } catch (error) {
72
+ controller.error(error);
73
+ finishOnce();
74
+ }
75
+ },
76
+ async cancel(reason) {
77
+ finishOnce();
78
+ await reader.cancel(reason);
79
+ },
80
+ });
81
+ return new Response(body, response);
82
+ }
83
+
21
84
  export async function handleResidentRead(
22
85
  runtime: ResidentRuntime,
23
86
  request: Request | undefined,
@@ -27,13 +90,28 @@ export async function handleResidentRead(
27
90
  if (!admitted) return unavailableResponse();
28
91
 
29
92
  let releaseReader: (() => void) | undefined;
93
+ let deferredFinish = false;
94
+ const isAuthorizationEpochCurrent = (): boolean =>
95
+ admitted.isAuthorizationEpochCurrent?.() ?? true;
96
+ const finish = (): void => {
97
+ releaseReader?.();
98
+ admitted.finish();
99
+ };
30
100
  try {
31
101
  releaseReader = await runtime.readerGate.acquire(admitted.signal);
32
102
  if (admitted.signal.aborted) return unavailableResponse();
33
103
  const response = await runtime.withModelLease(() =>
34
104
  Promise.resolve(operation(admitted.signal))
35
105
  );
36
- return admitted.signal.aborted ? unavailableResponse() : response;
106
+ if (admitted.signal.aborted) return unavailableResponse();
107
+ if (!isAuthorizationEpochCurrent()) {
108
+ return policyChangedResponse();
109
+ }
110
+ if (response.headers.get("content-type")?.includes("text/event-stream")) {
111
+ deferredFinish = true;
112
+ return wrapResidentStream(response, isAuthorizationEpochCurrent, finish);
113
+ }
114
+ return response;
37
115
  } catch (error) {
38
116
  if (
39
117
  admitted.signal.aborted ||
@@ -49,7 +127,6 @@ export async function handleResidentRead(
49
127
  }
50
128
  throw error;
51
129
  } finally {
52
- releaseReader?.();
53
- admitted.finish();
130
+ if (!deferredFinish) finish();
54
131
  }
55
132
  }
@@ -32,6 +32,8 @@ import {
32
32
  loadConfig,
33
33
  } from "../config";
34
34
  import { SavedCapsuleReverificationScheduler } from "../core/capsule-reverification-scheduler";
35
+ import { collectionEgressPolicyEpoch } from "../core/collection-egress-policy-service";
36
+ import { authorizeCurrentEgress } from "../core/egress-authorization";
35
37
  import { acquireWriteLock } from "../core/file-lock";
36
38
  import { JobManager } from "../core/job-manager";
37
39
  import { recordContentMutation } from "../core/mutation-generations";
@@ -78,8 +80,10 @@ export interface ResidentGeneration {
78
80
  }
79
81
 
80
82
  export interface ResidentRequestHandle {
83
+ authorizationEpoch?: string;
81
84
  id: string;
82
85
  signal: AbortSignal;
86
+ isAuthorizationEpochCurrent?(): boolean;
83
87
  finish(): void;
84
88
  }
85
89
 
@@ -101,12 +105,14 @@ export interface ResidentRuntime {
101
105
  readonly generations: ResidentGeneration;
102
106
  readonly activeRequests: number;
103
107
  readonly activeSessions: number;
108
+ readonly authorizationEpoch: string;
104
109
  readonly isShuttingDown: boolean;
105
110
  getStatus(): ResidentStatus;
106
111
  setListenerPort(port: number | null): void;
107
112
  setTransportStatusProvider(
108
113
  provider: (() => HttpMcpTransportStatus) | null
109
114
  ): void;
115
+ setPolicySessionInvalidator(invalidator: (() => Promise<void>) | null): void;
110
116
  admitRequest(signal?: AbortSignal): ResidentRequestHandle | null;
111
117
  withModelLease<T>(operation: () => Promise<T>): Promise<T>;
112
118
  markContentMutation(): void;
@@ -306,6 +312,8 @@ export async function startResidentRuntime(
306
312
  serverInstanceId,
307
313
  toolMutex,
308
314
  });
315
+ let authorizationEpoch = collectionEgressPolicyEpoch(initialConfig);
316
+ jobManager.setAuthorizationEpoch(authorizationEpoch);
309
317
  ctxHolder.jobManager = jobManager;
310
318
  const admission = new AdmissionController();
311
319
  const readerGate = new ReaderGate(
@@ -315,6 +323,7 @@ export async function startResidentRuntime(
315
323
  const startedAt = Date.now();
316
324
  let listenerPort: number | null = null;
317
325
  let transportStatusProvider: (() => HttpMcpTransportStatus) | null = null;
326
+ let policySessionInvalidator: (() => Promise<void>) | null = null;
318
327
  let shutdownState: ResidentStatus["shutdown"]["state"] = "none";
319
328
  let admissionState: ResidentStatus["admission"]["state"] = "accepting";
320
329
  let disposed = false;
@@ -359,7 +368,32 @@ export async function startResidentRuntime(
359
368
  markIndexMutation: () => {
360
369
  generations.index += 1;
361
370
  },
371
+ invalidateEgressPolicy: async () =>
372
+ (await ctxHolder.invalidateEgressPolicy?.()) ?? {
373
+ policyEpoch: collectionEgressPolicyEpoch(ctxHolder.config),
374
+ queuedJobsInvalidated: 0,
375
+ sessionsInvalidated: 0,
376
+ staleWorkMustRetry: true,
377
+ },
362
378
  });
379
+ mcpContext.authorizeTraceExport = async (lineage) => {
380
+ const egress = mcpContext.getEgressContext?.();
381
+ return authorizeCurrentEgress({
382
+ store,
383
+ config: ctxHolder.config,
384
+ lineage,
385
+ action: "export",
386
+ destinationZone:
387
+ egress?.destinationZone === "loopback"
388
+ ? "local_process"
389
+ : (egress?.destinationZone ?? "local_process"),
390
+ caller: egress?.caller ?? {
391
+ authenticated: true,
392
+ operationAuthorized: true,
393
+ },
394
+ contentClass: "retrieval_trace",
395
+ });
396
+ };
363
397
 
364
398
  const runtime: ResidentRuntime = {
365
399
  mode: options.mode ?? "serve",
@@ -385,10 +419,22 @@ export async function startResidentRuntime(
385
419
  get activeSessions() {
386
420
  return transportStatusProvider?.().activeSessions ?? 0;
387
421
  },
422
+ get authorizationEpoch() {
423
+ return authorizationEpoch;
424
+ },
388
425
  get isShuttingDown() {
389
426
  return disposed || !admission.accepting;
390
427
  },
391
- admitRequest: (signal) => admission.admit(signal),
428
+ admitRequest: (signal) => {
429
+ const admitted = admission.admit(signal);
430
+ if (!admitted) return null;
431
+ const requestEpoch = authorizationEpoch;
432
+ return {
433
+ ...admitted,
434
+ authorizationEpoch: requestEpoch,
435
+ isAuthorizationEpochCurrent: () => requestEpoch === authorizationEpoch,
436
+ };
437
+ },
392
438
  async withModelLease<T>(operation: () => Promise<T>): Promise<T> {
393
439
  const lease = modelManager.acquireLease();
394
440
  try {
@@ -450,6 +496,9 @@ export async function startResidentRuntime(
450
496
  setTransportStatusProvider(provider) {
451
497
  transportStatusProvider = provider;
452
498
  },
499
+ setPolicySessionInvalidator(invalidator) {
500
+ policySessionInvalidator = invalidator;
501
+ },
453
502
  async syncAll(syncOptions = {}) {
454
503
  const config = ctxHolder.config;
455
504
  const syncAllService = deps.syncAllService
@@ -508,6 +557,19 @@ export async function startResidentRuntime(
508
557
  };
509
558
  ctxHolder.startBackgroundWork = (operation) =>
510
559
  runtime.startBackgroundWork(operation);
560
+ ctxHolder.invalidateEgressPolicy = async () => {
561
+ const sessionsInvalidated = transportStatusProvider?.().activeSessions ?? 0;
562
+ const policyEpoch = collectionEgressPolicyEpoch(ctxHolder.config);
563
+ authorizationEpoch = policyEpoch;
564
+ const queuedJobsInvalidated = jobManager.setAuthorizationEpoch(policyEpoch);
565
+ await policySessionInvalidator?.();
566
+ return {
567
+ policyEpoch,
568
+ queuedJobsInvalidated,
569
+ sessionsInvalidated,
570
+ staleWorkMustRetry: true,
571
+ };
572
+ };
511
573
  mcpContext.getResidentStatus = () => runtime.getStatus();
512
574
  return { success: true, runtime };
513
575
  }