@chengxianglibra/dsh-data-analysis 0.1.1 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (195) hide show
  1. package/README.md +83 -34
  2. package/lib/bin/environment.js +1 -1
  3. package/lib/bin/presentation-lint.js +21 -0
  4. package/lib/bridges.js +2 -2
  5. package/lib/client/credentials/create-datasource.js +67 -0
  6. package/lib/client/credentials/install.js +191 -0
  7. package/lib/client/credentials/model.js +494 -0
  8. package/lib/client/credentials/styles.js +17 -0
  9. package/lib/client/workspace-header-action.js +11 -0
  10. package/lib/client.js +1133 -736
  11. package/lib/compatibility.js +1 -1
  12. package/lib/datasource/authoring.js +70 -0
  13. package/lib/datasource/bridge-programs.js +30 -18
  14. package/lib/datasource/bridge.js +100 -8
  15. package/lib/datasource/index.js +3 -2
  16. package/lib/datasource/python.js +115 -0
  17. package/lib/datasource/resolver-program.js +112 -0
  18. package/lib/datasource/rpc.js +88 -0
  19. package/lib/datasource/service.js +687 -0
  20. package/lib/datasource/shell-env.js +2 -333
  21. package/lib/datasource/test.js +24 -51
  22. package/lib/environment/binding.js +54 -9
  23. package/lib/environment/runtime.js +89 -78
  24. package/lib/environment/subprocess.js +6 -1
  25. package/lib/environment/summary.js +3 -3
  26. package/lib/environment/workspace.js +5 -0
  27. package/lib/index.js +1 -1
  28. package/lib/plugin.js +168 -51
  29. package/lib/presentation/assets/portable.js +606 -0
  30. package/lib/presentation/assets/static.js +945 -0
  31. package/lib/presentation/build/index.js +71 -0
  32. package/lib/presentation/commit.js +156 -0
  33. package/lib/presentation/contracts/charts.js +476 -0
  34. package/lib/presentation/contracts/editing.js +58 -0
  35. package/lib/presentation/contracts/index.js +651 -0
  36. package/lib/presentation/contracts/interaction.js +62 -0
  37. package/lib/presentation/contracts/types.js +15 -0
  38. package/lib/presentation/contracts/values.js +28 -0
  39. package/lib/presentation/delivery.js +56 -0
  40. package/lib/presentation/files.js +83 -0
  41. package/lib/presentation/index.js +6 -0
  42. package/lib/presentation/lint.js +46 -0
  43. package/lib/presentation/projection/files.js +85 -0
  44. package/lib/presentation/projection/index.js +195 -0
  45. package/lib/presentation/projection/program.js +144 -0
  46. package/lib/presentation/projection/sql-code.js +79 -0
  47. package/lib/presentation/receipt.js +34 -0
  48. package/lib/presentation/reports.js +122 -0
  49. package/lib/presentation/rpc.js +148 -0
  50. package/lib/presentation/tool.js +122 -0
  51. package/lib/python-execution-contracts.js +1 -0
  52. package/lib/python-execution.js +232 -0
  53. package/lib/semantic-browser/contracts.js +80 -0
  54. package/lib/semantic-browser/definition.js +64 -0
  55. package/lib/semantic-browser/program.js +119 -0
  56. package/lib/semantic-browser/service.js +72 -0
  57. package/lib/semantic-reference/bridge.js +118 -0
  58. package/lib/semantic-reference/contracts.js +113 -0
  59. package/lib/semantic-reference/index.js +3 -0
  60. package/lib/semantic-reference/rpc.js +122 -0
  61. package/lib/semantic-reference/search.js +94 -0
  62. package/lib/semantic-reference/usage.js +143 -0
  63. package/lib/types/bridges.d.ts +2 -2
  64. package/lib/types/client/credentials/create-datasource.d.ts +5 -0
  65. package/lib/types/client/credentials/install.d.ts +1 -0
  66. package/lib/types/client/credentials/model.d.ts +59 -0
  67. package/lib/types/client/credentials/styles.d.ts +1 -0
  68. package/lib/types/client/presentation/ask-dsh.d.ts +9 -0
  69. package/lib/types/client/presentation/chart-explorer.d.ts +12 -0
  70. package/lib/types/client/presentation/chart-geometry.d.ts +30 -0
  71. package/lib/types/client/presentation/chart-renderer.d.ts +10 -0
  72. package/lib/types/client/presentation/chart-specials.d.ts +10 -0
  73. package/lib/types/client/presentation/chart-styles.d.ts +1 -0
  74. package/lib/types/client/presentation/chart-tooltip.d.ts +13 -0
  75. package/lib/types/client/presentation/chart-view.d.ts +23 -0
  76. package/lib/types/client/presentation/copy-context.d.ts +5 -0
  77. package/lib/types/client/presentation/delivery-model.d.ts +54 -0
  78. package/lib/types/client/presentation/delivery.d.ts +77 -0
  79. package/lib/types/client/presentation/editor-controls.d.ts +12 -0
  80. package/lib/types/client/presentation/global-filters.d.ts +6 -0
  81. package/lib/types/client/presentation/host-entry.d.ts +12 -0
  82. package/lib/types/client/presentation/icons.d.ts +4 -0
  83. package/lib/types/client/presentation/install.d.ts +15 -0
  84. package/lib/types/client/presentation/markdown.d.ts +5 -0
  85. package/lib/types/client/presentation/model.d.ts +62 -0
  86. package/lib/types/client/presentation/reader.d.ts +11 -0
  87. package/lib/types/client/presentation/source-code-model.d.ts +16 -0
  88. package/lib/types/client/presentation/source-code.d.ts +11 -0
  89. package/lib/types/client/presentation/source-dialog.d.ts +13 -0
  90. package/lib/types/client/presentation/source-facts.d.ts +20 -0
  91. package/lib/types/client/presentation/source-format.d.ts +5 -0
  92. package/lib/types/client/presentation/source-highlight.d.ts +7 -0
  93. package/lib/types/client/presentation/sources.d.ts +13 -0
  94. package/lib/types/client/presentation/styles.d.ts +1 -0
  95. package/lib/types/client/presentation/table.d.ts +12 -0
  96. package/lib/types/client/presentation/use-element-width.d.ts +5 -0
  97. package/lib/types/client/workspace-header-action.d.ts +7 -0
  98. package/lib/types/client.d.ts +4 -122
  99. package/lib/types/compatibility.d.ts +2 -2
  100. package/lib/types/datasource/authoring.d.ts +25 -0
  101. package/lib/types/datasource/bridge.d.ts +21 -3
  102. package/lib/types/datasource/index.d.ts +3 -5
  103. package/lib/types/datasource/python.d.ts +4 -0
  104. package/lib/types/datasource/resolver-program.d.ts +6 -0
  105. package/lib/types/datasource/rpc.d.ts +4 -0
  106. package/lib/types/datasource/service.d.ts +112 -0
  107. package/lib/types/datasource/shell-env.d.ts +0 -29
  108. package/lib/types/datasource/test.d.ts +6 -29
  109. package/lib/types/environment/binding.d.ts +5 -1
  110. package/lib/types/environment/errors.d.ts +1 -1
  111. package/lib/types/environment/runtime.d.ts +5 -5
  112. package/lib/types/environment/types.d.ts +13 -2
  113. package/lib/types/index.d.ts +1 -1
  114. package/lib/types/plugin.d.ts +11 -5
  115. package/lib/types/presentation/build/index.d.ts +11 -0
  116. package/lib/types/presentation/commit.d.ts +4 -0
  117. package/lib/types/presentation/contracts/charts.d.ts +13 -0
  118. package/lib/types/presentation/contracts/editing.d.ts +8 -0
  119. package/lib/types/presentation/contracts/index.d.ts +11 -0
  120. package/lib/types/presentation/contracts/interaction.d.ts +8 -0
  121. package/lib/types/presentation/contracts/types.d.ts +252 -0
  122. package/lib/types/presentation/contracts/values.d.ts +9 -0
  123. package/lib/types/presentation/delivery.d.ts +3 -0
  124. package/lib/types/presentation/files.d.ts +7 -0
  125. package/lib/types/presentation/index.d.ts +6 -0
  126. package/lib/types/presentation/lint.d.ts +13 -0
  127. package/lib/types/presentation/projection/files.d.ts +2 -0
  128. package/lib/types/presentation/projection/index.d.ts +20 -0
  129. package/lib/types/presentation/projection/program.d.ts +2 -0
  130. package/lib/types/presentation/projection/sql-code.d.ts +2 -0
  131. package/lib/types/presentation/receipt.d.ts +14 -0
  132. package/lib/types/presentation/reports.d.ts +5 -0
  133. package/lib/types/presentation/rpc.d.ts +24 -0
  134. package/lib/types/presentation/tool.d.ts +12 -0
  135. package/lib/types/python-execution-contracts.d.ts +10 -0
  136. package/lib/types/python-execution.d.ts +10 -0
  137. package/lib/types/semantic-browser/contracts.d.ts +40 -0
  138. package/lib/types/semantic-browser/definition.d.ts +20 -0
  139. package/lib/types/semantic-browser/program.d.ts +2 -0
  140. package/lib/types/semantic-browser/service.d.ts +20 -0
  141. package/lib/types/semantic-reference/bridge.d.ts +11 -0
  142. package/lib/types/semantic-reference/contracts.d.ts +51 -0
  143. package/lib/types/semantic-reference/index.d.ts +3 -0
  144. package/lib/types/semantic-reference/rpc.d.ts +15 -0
  145. package/lib/types/semantic-reference/search.d.ts +10 -0
  146. package/lib/types/semantic-reference/usage.d.ts +43 -0
  147. package/package.json +43 -19
  148. package/python/presentation-kit/dist/dsh_data_analysis_presentation_kit-1.1.0-py3-none-any.whl +0 -0
  149. package/skills/dsh-data-analysis-presentation/SKILL.md +24 -0
  150. package/skills/dsh-data-analysis-presentation/references/chart-examples.md +18 -0
  151. package/skills/dsh-data-analysis-presentation/references/charts.md +55 -0
  152. package/skills/dsh-data-analysis-presentation/references/examples/artifact.draft.json +27 -0
  153. package/skills/dsh-data-analysis-presentation/references/examples/charts-bins.dataset.json +33 -0
  154. package/skills/dsh-data-analysis-presentation/references/examples/charts-boxes.dataset.json +45 -0
  155. package/skills/dsh-data-analysis-presentation/references/examples/charts-composition.dataset.json +33 -0
  156. package/skills/dsh-data-analysis-presentation/references/examples/charts-funnel.dataset.json +27 -0
  157. package/skills/dsh-data-analysis-presentation/references/examples/charts-precision.dataset.json +21 -0
  158. package/skills/dsh-data-analysis-presentation/references/examples/charts-series.dataset.json +58 -0
  159. package/skills/dsh-data-analysis-presentation/references/examples/charts-shares.dataset.json +33 -0
  160. package/skills/dsh-data-analysis-presentation/references/examples/charts-waterfall.dataset.json +44 -0
  161. package/skills/dsh-data-analysis-presentation/references/examples/charts.draft.json +392 -0
  162. package/skills/dsh-data-analysis-presentation/references/examples/comparison-selected.dataset.json +24 -0
  163. package/skills/dsh-data-analysis-presentation/references/examples/comparison-summary.dataset.json +22 -0
  164. package/skills/dsh-data-analysis-presentation/references/examples/comparison.dataset.json +18 -0
  165. package/skills/dsh-data-analysis-presentation/references/examples/comparison.draft.json +79 -0
  166. package/skills/dsh-data-analysis-presentation/references/examples/computed-sources.draft.json +35 -0
  167. package/skills/dsh-data-analysis-presentation/references/examples/computed.dataset.json +12 -0
  168. package/skills/dsh-data-analysis-presentation/references/examples/computed.draft.json +57 -0
  169. package/skills/dsh-data-analysis-presentation/references/examples/interaction-detail.dataset.json +30 -0
  170. package/skills/dsh-data-analysis-presentation/references/examples/interaction-summary.dataset.json +22 -0
  171. package/skills/dsh-data-analysis-presentation/references/examples/interaction.draft.json +278 -0
  172. package/skills/dsh-data-analysis-presentation/references/examples/source-only.draft.json +19 -0
  173. package/skills/dsh-data-analysis-presentation/references/examples/write-charts.py +73 -0
  174. package/skills/dsh-data-analysis-presentation/references/examples/write-comparison.py +42 -0
  175. package/skills/dsh-data-analysis-presentation/references/examples/write-computed.py +21 -0
  176. package/skills/dsh-data-analysis-presentation/references/examples/write-interaction.py +33 -0
  177. package/skills/dsh-data-analysis-presentation/references/examples.md +42 -0
  178. package/skills/dsh-data-analysis-presentation/references/interaction.md +56 -0
  179. package/skills/dsh-data-analysis-presentation/references/narrative.md +37 -0
  180. package/skills/dsh-data-analysis-presentation/references/schema.md +138 -0
  181. package/lib/datasource/access.js +0 -60
  182. package/lib/datasource/credentials.js +0 -24
  183. package/lib/evidence/bridge-program.js +0 -153
  184. package/lib/evidence/bridge.js +0 -166
  185. package/lib/evidence/index.js +0 -2
  186. package/lib/evidence/sources.js +0 -263
  187. package/lib/types/datasource/access.d.ts +0 -27
  188. package/lib/types/evidence/bridge.d.ts +0 -40
  189. package/lib/types/evidence/index.d.ts +0 -4
  190. package/lib/types/evidence/sources.d.ts +0 -57
  191. package/python/report-kit/dist/dsh_data_analysis_report_kit-3.0.0-py3-none-any.whl +0 -0
  192. package/skills/dsh-data-analysis-report/SKILL.md +0 -57
  193. package/skills/dsh-data-analysis-report/assets/marivo-artifact.js +0 -113
  194. package/skills/dsh-data-analysis-report/assets/marivo-session-dag.js +0 -1201
  195. package/skills/dsh-data-analysis-report/assets/report-data.js +0 -449
@@ -0,0 +1,118 @@
1
+ import { parseProjection } from "./contracts.js";
2
+ export const CATALOG_PROGRAM = String.raw `
3
+ import contextlib, io, json, sys
4
+ with contextlib.redirect_stdout(io.StringIO()):
5
+ import marivo.semantic as ms
6
+ from marivo.refs import RefPayloadV1, SemanticKind
7
+ catalog = ms.load(workspace_dir=sys.argv[1])
8
+ rows = []
9
+ for kind in SemanticKind:
10
+ for entry in catalog.items(kind).items:
11
+ definition = entry.details().context.business_definition
12
+ rows.append({"ref": RefPayloadV1.from_ref(entry.ref).to_dict(), "refKey": entry.ref.key,
13
+ "name": entry.name, "businessDefinition": definition[:240] if definition else None})
14
+ result = {"kinds": [kind.value for kind in SemanticKind], "items": rows}
15
+ print(json.dumps(result, ensure_ascii=True))
16
+ `.trim();
17
+ /** One plugin-owned cache, keyed by the exact Environment fingerprint. */
18
+ export class SemanticReferenceBridge {
19
+ #slots = new Map();
20
+ #clock;
21
+ #disposed = false;
22
+ constructor(clock = Date.now) {
23
+ this.#clock = clock;
24
+ }
25
+ async candidates(runner, signal) {
26
+ signal.throwIfAborted();
27
+ if (this.#disposed)
28
+ throw new Error('disposed');
29
+ const key = runner.binding.fingerprint;
30
+ if (runner.status !== 'ready') {
31
+ this.invalidate(key);
32
+ throw new Error('environment-failed');
33
+ }
34
+ let slot = this.#slots.get(key);
35
+ if (!slot) {
36
+ slot = {};
37
+ this.#slots.set(key, slot);
38
+ }
39
+ if (slot.snapshot && this.#clock() < slot.expires)
40
+ return slot.snapshot;
41
+ if (!slot.flight) {
42
+ const current = slot;
43
+ const controller = new AbortController();
44
+ const flight = {
45
+ controller,
46
+ waiters: 0,
47
+ promise: Promise.resolve({ kinds: [], items: [] }),
48
+ };
49
+ current.flight = flight;
50
+ flight.promise = this.#load(runner, controller.signal)
51
+ .then((projection) => {
52
+ if (!controller.signal.aborted &&
53
+ !this.#disposed &&
54
+ current.flight === flight &&
55
+ runner.status === 'ready') {
56
+ current.snapshot = projection;
57
+ current.expires = this.#clock() + 30_000;
58
+ }
59
+ return projection;
60
+ })
61
+ .finally(() => {
62
+ if (current.flight === flight)
63
+ current.flight = undefined;
64
+ });
65
+ }
66
+ const current = slot, flight = slot.flight;
67
+ flight.waiters++;
68
+ return new Promise((resolve, reject) => {
69
+ let settled = false;
70
+ const finish = (value, error) => {
71
+ if (settled)
72
+ return;
73
+ settled = true;
74
+ signal.removeEventListener('abort', abort);
75
+ flight.controller.signal.removeEventListener('abort', loadAbort);
76
+ flight.waiters--;
77
+ if (!flight.waiters && current.flight === flight) {
78
+ current.flight = undefined;
79
+ flight.controller.abort();
80
+ }
81
+ if (error !== undefined)
82
+ reject(error);
83
+ else
84
+ resolve(value);
85
+ };
86
+ const abort = () => finish(undefined, new Error('cancelled'));
87
+ const loadAbort = () => finish(undefined, new Error('cancelled'));
88
+ signal.addEventListener('abort', abort, { once: true });
89
+ flight.controller.signal.addEventListener('abort', loadAbort, { once: true });
90
+ flight.promise.then((value) => finish(value), (error) => finish(undefined, error));
91
+ if (signal.aborted || flight.controller.signal.aborted)
92
+ abort();
93
+ });
94
+ }
95
+ async #load(runner, signal) {
96
+ const result = await runner.runChecked({
97
+ program: CATALOG_PROGRAM,
98
+ args: [runner.binding.projectRoot],
99
+ signal,
100
+ environmentOverlay: { MARIVO_TELEMETRY: 'off', PYTHONDONTWRITEBYTECODE: '1' },
101
+ limits: { timeoutMs: 30_000, stdoutMaxBytes: 32 * 1024 * 1024, stderrMaxBytes: 8192 },
102
+ });
103
+ signal.throwIfAborted();
104
+ if (result.exitCode !== 0)
105
+ throw new Error('catalog-load-failed');
106
+ return parseProjection(JSON.parse(result.stdout.toString('utf8')));
107
+ }
108
+ invalidate(key) {
109
+ const slot = this.#slots.get(key);
110
+ this.#slots.delete(key);
111
+ slot?.flight?.controller.abort();
112
+ }
113
+ dispose() {
114
+ this.#disposed = true;
115
+ for (const key of this.#slots.keys())
116
+ this.invalidate(key);
117
+ }
118
+ }
@@ -0,0 +1,113 @@
1
+ /** Browser-safe wire contracts. These bounds do not reimplement Marivo's domain grammar. */
2
+ export const CHANNEL = '/dsh-data-analysis';
3
+ export const SOURCE = 'marivo-semantic';
4
+ export const LIMIT = 40;
5
+ export const MAX_WIRE_BYTES = 1024 * 1024;
6
+ export function closed(value, keys) {
7
+ if (typeof value !== 'object' || value === null || Array.isArray(value))
8
+ throw new Error('invalid-request');
9
+ const record = value;
10
+ if (Object.keys(record).length !== keys.length || keys.some((key) => !Object.hasOwn(record, key)))
11
+ throw new Error('invalid-request');
12
+ return record;
13
+ }
14
+ export function boundedText(value, max, empty = false) {
15
+ if (typeof value !== 'string' ||
16
+ value.length > max * 2 ||
17
+ (!empty && value.length === 0) ||
18
+ Array.from(value).length > max)
19
+ throw new Error('invalid-request');
20
+ return value;
21
+ }
22
+ export function parseJson(value, max = 16_384) {
23
+ const text = boundedText(value, max);
24
+ return JSON.parse(text);
25
+ }
26
+ export function parseRef(value) {
27
+ const r = closed(value, ['schema', 'kind', 'path']);
28
+ if (r.schema !== 'marivo.semantic_ref/v1')
29
+ throw new Error('invalid-request');
30
+ return Object.freeze({
31
+ schema: r.schema,
32
+ kind: boundedText(r.kind, 128),
33
+ path: boundedText(r.path, 2048),
34
+ });
35
+ }
36
+ export function refKey(ref) {
37
+ return `${ref.kind}:${ref.path}`;
38
+ }
39
+ export function parseEnvelope(value) {
40
+ const r = closed(value, ['schema', 'sessionId', 'environmentFingerprint', 'ref']);
41
+ if (r.schema !== 'dsh-data-analysis-semantic-reference/v1')
42
+ throw new Error('invalid-request');
43
+ return Object.freeze({
44
+ schema: r.schema,
45
+ sessionId: boundedText(r.sessionId, 256),
46
+ environmentFingerprint: boundedText(r.environmentFingerprint, 256),
47
+ ref: parseRef(r.ref),
48
+ });
49
+ }
50
+ export function envelopeJson(envelope) {
51
+ return JSON.stringify(parseEnvelope(envelope));
52
+ }
53
+ export function modelMarker(envelope) {
54
+ // Keep JSON inside one marker even when a valid domain path contains markup characters.
55
+ const json = JSON.stringify(parseRef(envelope.ref))
56
+ .replaceAll('<', '\\u003c')
57
+ .replaceAll('>', '\\u003e')
58
+ .replaceAll('&', '\\u0026');
59
+ return `<marivo-semantic-ref>${json}</marivo-semantic-ref>`;
60
+ }
61
+ export function parseCandidate(value) {
62
+ const r = closed(value, ['ref', 'refKey', 'name', 'businessDefinition']);
63
+ const ref = parseRef(r.ref);
64
+ if (r.refKey !== refKey(ref))
65
+ throw new Error('invalid-request');
66
+ return Object.freeze({
67
+ ref,
68
+ refKey: r.refKey,
69
+ name: boundedText(r.name, 2048),
70
+ businessDefinition: r.businessDefinition === null ? null : boundedText(r.businessDefinition, 240, true),
71
+ });
72
+ }
73
+ export function parseProjection(value) {
74
+ const r = closed(value, ['kinds', 'items']);
75
+ if (!Array.isArray(r.kinds) ||
76
+ r.kinds.length > 128 ||
77
+ !Array.isArray(r.items) ||
78
+ r.items.length > 100_000)
79
+ throw new Error('invalid-request');
80
+ const kinds = r.kinds.map((kind) => boundedText(kind, 128));
81
+ if (new Set(kinds).size !== kinds.length)
82
+ throw new Error('invalid-request');
83
+ const items = r.items.map(parseCandidate);
84
+ if (new Set(items.map((item) => item.refKey)).size !== items.length ||
85
+ items.some((item) => !kinds.includes(item.ref.kind)))
86
+ throw new Error('invalid-request');
87
+ return Object.freeze({ kinds: Object.freeze(kinds), items: Object.freeze(items) });
88
+ }
89
+ export function parseCandidatesRequest(value) {
90
+ const r = closed(value, ['version', 'sessionId', 'query', 'quoted', 'limit']);
91
+ if (r.version !== 1 || r.limit !== LIMIT || typeof r.quoted !== 'boolean')
92
+ throw new Error('invalid-request');
93
+ return {
94
+ version: 1,
95
+ sessionId: boundedText(r.sessionId, 256),
96
+ query: boundedText(r.query, 128, true),
97
+ quoted: r.quoted,
98
+ limit: LIMIT,
99
+ };
100
+ }
101
+ export function parseCandidatesResponse(value) {
102
+ const r = closed(value, ['environmentFingerprint', 'items']);
103
+ if (!Array.isArray(r.items) || r.items.length > LIMIT)
104
+ throw new Error('invalid-response');
105
+ const items = r.items.map((raw) => {
106
+ const item = closed(raw, ['ref', 'refKey', 'name', 'businessDefinition', 'section']);
107
+ const { section, ...candidate } = item;
108
+ if (section !== 'recent' && section !== 'kind' && section !== 'strict' && section !== 'fuzzy')
109
+ throw new Error('invalid-response');
110
+ return { ...parseCandidate(candidate), section };
111
+ });
112
+ return { environmentFingerprint: boundedText(r.environmentFingerprint, 256), items };
113
+ }
@@ -0,0 +1,3 @@
1
+ export { SemanticReferenceBridge } from "./bridge.js";
2
+ export { registerSemanticReferenceRpc, SemanticReferenceService } from "./rpc.js";
3
+ export { SemanticReferenceUsage } from "./usage.js";
@@ -0,0 +1,122 @@
1
+ import { SemanticReferenceBridge } from "./bridge.js";
2
+ import { CHANNEL, closed, MAX_WIRE_BYTES, modelMarker, parseCandidatesRequest, parseEnvelope, } from "./contracts.js";
3
+ import { search } from "./search.js";
4
+ import { workspaceKey } from "./usage.js";
5
+ /** Abort the caller's wait, without pretending to cancel the shared Environment resolver. */
6
+ export async function abortable(promise, signal) {
7
+ return new Promise((resolve, reject) => {
8
+ const abort = () => {
9
+ signal.removeEventListener('abort', abort);
10
+ reject(new Error('cancelled'));
11
+ };
12
+ signal.addEventListener('abort', abort, { once: true });
13
+ promise.then((value) => {
14
+ signal.removeEventListener('abort', abort);
15
+ if (signal.aborted)
16
+ reject(new Error('cancelled'));
17
+ else
18
+ resolve(value);
19
+ }, (error) => {
20
+ signal.removeEventListener('abort', abort);
21
+ reject(error);
22
+ });
23
+ if (signal.aborted)
24
+ abort();
25
+ });
26
+ }
27
+ export class SemanticReferenceService {
28
+ #resolve;
29
+ #usage;
30
+ #bridge;
31
+ #controller = new AbortController();
32
+ #closed = false;
33
+ constructor(resolve, usage, bridge = new SemanticReferenceBridge()) {
34
+ this.#resolve = resolve;
35
+ this.#usage = usage;
36
+ this.#bridge = bridge;
37
+ }
38
+ async #environment(sessionId, signal, purpose = 'reference') {
39
+ const runner = await abortable(this.#resolve(sessionId, purpose), signal);
40
+ if (runner.status !== 'ready') {
41
+ this.#bridge.invalidate(runner.binding.fingerprint);
42
+ throw new Error('environment-failed');
43
+ }
44
+ return runner;
45
+ }
46
+ async #owner(envelope, signal) {
47
+ const runner = await this.#environment(envelope.sessionId, signal);
48
+ if (runner.binding.fingerprint !== envelope.environmentFingerprint)
49
+ throw new Error('environment-mismatch');
50
+ return runner;
51
+ }
52
+ async handle(endpoint, payload, caller) {
53
+ if (this.#closed)
54
+ throw new Error('disposed');
55
+ const signal = AbortSignal.any([caller, this.#controller.signal]);
56
+ signal.throwIfAborted();
57
+ if (Buffer.byteLength(JSON.stringify(payload) ?? '') > MAX_WIRE_BYTES)
58
+ throw new Error('invalid-request');
59
+ if (endpoint === 'semantic-references/candidates') {
60
+ const request = parseCandidatesRequest(payload);
61
+ const environment = await this.#environment(request.sessionId, signal, 'candidates');
62
+ const projection = await this.#bridge.candidates(environment, signal);
63
+ const scores = await abortable(this.#usage.scores(workspaceKey(environment.binding.projectRoot)), signal);
64
+ // A live Agent may disappear or change binding while Catalog/storage work awaits.
65
+ const current = await this.#environment(request.sessionId, signal);
66
+ if (current.binding.fingerprint !== environment.binding.fingerprint)
67
+ throw new Error('environment-mismatch');
68
+ return {
69
+ environmentFingerprint: environment.binding.fingerprint,
70
+ items: search(projection, request.query, scores),
71
+ };
72
+ }
73
+ if (endpoint === 'semantic-references/selected' ||
74
+ endpoint === 'semantic-references/serialize') {
75
+ const request = closed(payload, ['envelope']), envelope = parseEnvelope(request.envelope);
76
+ const environment = await this.#owner(envelope, signal);
77
+ if (endpoint === 'semantic-references/serialize')
78
+ return { text: modelMarker(envelope) };
79
+ await this.#usage.selected(workspaceKey(environment.binding.projectRoot), envelope.ref, signal);
80
+ return { selected: true };
81
+ }
82
+ throw new Error('unknown-endpoint');
83
+ }
84
+ stop() {
85
+ this.#closed = true;
86
+ this.#controller.abort();
87
+ this.#bridge.dispose();
88
+ }
89
+ async close() {
90
+ this.stop();
91
+ await this.#usage.close();
92
+ }
93
+ }
94
+ export function registerSemanticReferenceRpc(connection, service, browser) {
95
+ const unregister = connection.rpc.handle(CHANNEL, async (endpoint, payload, signal) => {
96
+ if (browser && endpoint === 'semantic-browser/catalog') {
97
+ return browser(endpoint, payload, signal);
98
+ }
99
+ try {
100
+ return { ok: true, value: await service.handle(endpoint, payload, signal) };
101
+ }
102
+ catch {
103
+ return {
104
+ ok: false,
105
+ error: {
106
+ code: 'internal',
107
+ message: 'Marivo semantic reference request failed',
108
+ details: {},
109
+ },
110
+ };
111
+ }
112
+ }, { authority: 'trusted-host' });
113
+ return async () => {
114
+ service.stop();
115
+ try {
116
+ await unregister();
117
+ }
118
+ finally {
119
+ await service.close();
120
+ }
121
+ };
122
+ }
@@ -0,0 +1,94 @@
1
+ import { LIMIT } from "./contracts.js";
2
+ export function normalize(text) {
3
+ return text.normalize('NFKC').toLowerCase().trim().replace(/\s+/gu, ' ');
4
+ }
5
+ function tokens(text) {
6
+ return text.split(/[\s.:_-]+/u).filter(Boolean);
7
+ }
8
+ export function compareText(a, b) {
9
+ return a < b ? -1 : a > b ? 1 : 0;
10
+ }
11
+ function length(text) {
12
+ return Array.from(text).length;
13
+ }
14
+ export function dice(a, b) {
15
+ const grams = (text) => {
16
+ const chars = Array.from(text);
17
+ return new Set(chars.slice(0, -2).map((_, i) => chars.slice(i, i + 3).join('')));
18
+ };
19
+ const left = grams(a), right = grams(b);
20
+ if (!left.size || !right.size)
21
+ return 0;
22
+ return (2 * [...left].filter((gram) => right.has(gram)).length) / (left.size + right.size);
23
+ }
24
+ function match(item, query) {
25
+ const name = normalize(item.name), path = normalize(item.ref.path), key = normalize(item.refKey);
26
+ const fields = [name, path, key];
27
+ const all = [...fields, normalize(item.ref.kind)];
28
+ const searchTokens = all.flatMap(tokens);
29
+ if (fields.includes(query))
30
+ return { item, tier: 0, score: 1, field: 0 };
31
+ const prefixes = [...fields, ...searchTokens]
32
+ .map((value, field) => ({ value, field }))
33
+ .filter(({ value }) => value.startsWith(query));
34
+ if (prefixes.length) {
35
+ prefixes.sort((a, b) => length(a.value) - length(b.value) || a.field - b.field);
36
+ const best = prefixes[0];
37
+ return {
38
+ item,
39
+ tier: 1,
40
+ score: length(query) / length(best.value),
41
+ field: best.field < 3 ? best.field : 3,
42
+ };
43
+ }
44
+ const document = [...all, normalize(item.businessDefinition ?? '')].join(' ');
45
+ const qt = tokens(query);
46
+ if (qt.length && qt.every((token) => document.includes(token)))
47
+ return {
48
+ item,
49
+ tier: 2,
50
+ score: qt.reduce((sum, token) => sum + length(token), 0) / Math.max(1, length(document)),
51
+ field: 0,
52
+ };
53
+ return {
54
+ item,
55
+ tier: 3,
56
+ score: length(query) < 3
57
+ ? 0
58
+ : Math.max(...[name, ...tokens(path), path].map((value) => dice(query, value))),
59
+ field: 0,
60
+ };
61
+ }
62
+ export function search(projection, rawQuery, usage = new Map()) {
63
+ const query = normalize(rawQuery);
64
+ const heat = (a, b) => (usage.get(b.refKey)?.count ?? 0) - (usage.get(a.refKey)?.count ?? 0) ||
65
+ (usage.get(b.refKey)?.last ?? 0) - (usage.get(a.refKey)?.last ?? 0);
66
+ const fallback = (a, b) => projection.kinds.indexOf(a.ref.kind) - projection.kinds.indexOf(b.ref.kind) ||
67
+ compareText(a.refKey, b.refKey);
68
+ if (!query) {
69
+ const recent = projection.items
70
+ .filter((item) => (usage.get(item.refKey)?.count ?? 0) > 0)
71
+ .sort((a, b) => heat(a, b) || fallback(a, b))
72
+ .slice(0, 10);
73
+ const keys = new Set(recent.map((item) => item.refKey));
74
+ return [
75
+ ...recent.map((item) => ({ ...item, section: 'recent' })),
76
+ ...projection.items
77
+ .filter((item) => !keys.has(item.refKey))
78
+ .sort(fallback)
79
+ .slice(0, LIMIT - recent.length)
80
+ .map((item) => ({ ...item, section: 'kind' })),
81
+ ];
82
+ }
83
+ const matches = projection.items.map((item) => match(item, query));
84
+ const strictCount = matches.filter((item) => item.tier < 3).length;
85
+ return matches
86
+ .filter((m) => m.tier < 3 || (strictCount < 12 && m.score >= 0.42))
87
+ .sort((a, b) => a.tier - b.tier ||
88
+ b.score - a.score ||
89
+ a.field - b.field ||
90
+ heat(a.item, b.item) ||
91
+ compareText(a.item.refKey, b.item.refKey))
92
+ .slice(0, LIMIT)
93
+ .map((m) => ({ ...m.item, section: m.tier === 3 ? 'fuzzy' : 'strict' }));
94
+ }
@@ -0,0 +1,143 @@
1
+ import { createHash } from 'node:crypto';
2
+ import { defineDomain, domainTable, } from '@deepseek-ai/dsh-storage-domain';
3
+ import { z } from 'zod';
4
+ import { parseRef, refKey } from "./contracts.js";
5
+ const refSchema = z.unknown().transform((value, ctx) => {
6
+ try {
7
+ return parseRef(value);
8
+ }
9
+ catch {
10
+ ctx.addIssue({ code: 'custom', message: 'invalid semantic ref' });
11
+ return z.NEVER;
12
+ }
13
+ });
14
+ const daySchema = z.strictObject({
15
+ day: z.iso.date(),
16
+ count: z.number().int().min(1).max(Number.MAX_SAFE_INTEGER),
17
+ });
18
+ export const usageSchema = z
19
+ .strictObject({
20
+ entries: z.record(z.string(), z.strictObject({
21
+ ref: refSchema,
22
+ days: z
23
+ .array(daySchema)
24
+ .min(1)
25
+ .max(7)
26
+ .refine((days) => days.every((day, i) => i === 0 || days[i - 1].day < day.day)),
27
+ lastSelectedAt: z.number().int().min(0).max(8_640_000_000_000_000),
28
+ })),
29
+ })
30
+ .refine((row) => Object.entries(row.entries).every(([key, entry]) => key === refKey(entry.ref)));
31
+ export const usageSpec = defineDomain({
32
+ name: 'dsh_data_analysis_semantic_reference_usage',
33
+ version: 0,
34
+ tables: { workspaces: domainTable(usageSchema) },
35
+ });
36
+ export function workspaceKey(canonicalRoot) {
37
+ return createHash('sha256').update(canonicalRoot).digest('hex');
38
+ }
39
+ export function localDay(date) {
40
+ return `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, '0')}-${String(date.getDate()).padStart(2, '0')}`;
41
+ }
42
+ export function windowDays(now) {
43
+ const date = new Date(now);
44
+ date.setHours(12, 0, 0, 0);
45
+ return Array.from({ length: 7 }, (_, i) => {
46
+ const d = new Date(date);
47
+ d.setDate(d.getDate() - (6 - i));
48
+ return localDay(d);
49
+ });
50
+ }
51
+ export function increment(row, input, now) {
52
+ const ref = parseRef(input), window = windowDays(now), today = window[6];
53
+ const entries = Object.create(null);
54
+ for (const [key, value] of Object.entries(row.entries)) {
55
+ const days = value.days.filter((day) => window.includes(day.day));
56
+ if (days.length)
57
+ entries[key] = { ...value, days };
58
+ }
59
+ const key = refKey(ref), prior = entries[key], days = prior?.days ?? [];
60
+ const count = days.find((day) => day.day === today)?.count ?? 0;
61
+ entries[key] = {
62
+ ref,
63
+ days: [
64
+ ...days.filter((day) => day.day !== today),
65
+ { day: today, count: Math.min(Number.MAX_SAFE_INTEGER, count + 1) },
66
+ ].sort((a, b) => a.day.localeCompare(b.day)),
67
+ lastSelectedAt: now,
68
+ };
69
+ return { entries };
70
+ }
71
+ /** Only admitted mutations join the per-Workspace queue; close drains those jobs. */
72
+ export class SemanticReferenceUsage {
73
+ #ready;
74
+ #queues = new Map();
75
+ #clock;
76
+ #diagnostic;
77
+ #closed = false;
78
+ constructor(facility, clock = Date.now, diagnostic = () => { }) {
79
+ this.#clock = clock;
80
+ this.#diagnostic = diagnostic;
81
+ this.#ready = Promise.resolve()
82
+ .then(() => facility.open(usageSpec))
83
+ .catch(() => {
84
+ diagnostic();
85
+ return undefined;
86
+ });
87
+ }
88
+ async scores(key) {
89
+ try {
90
+ const domain = await this.#ready;
91
+ if (!domain || this.#closed)
92
+ return new Map();
93
+ const window = windowDays(this.#clock()), row = domain.table('workspaces').get(key);
94
+ return new Map(Object.entries(row?.entries ?? {}).map(([key, entry]) => [
95
+ key,
96
+ {
97
+ count: entry.days
98
+ .filter((day) => window.includes(day.day))
99
+ .reduce((sum, day) => Math.min(Number.MAX_SAFE_INTEGER, sum + day.count), 0),
100
+ last: entry.lastSelectedAt,
101
+ },
102
+ ]));
103
+ }
104
+ catch {
105
+ this.#diagnostic();
106
+ return new Map();
107
+ }
108
+ }
109
+ async selected(key, ref, signal) {
110
+ const parsed = parseRef(ref);
111
+ const domain = await this.#ready;
112
+ signal.throwIfAborted();
113
+ if (this.#closed)
114
+ throw new Error('disposed');
115
+ if (!domain)
116
+ return;
117
+ const now = this.#clock();
118
+ const job = (this.#queues.get(key) ?? Promise.resolve())
119
+ .then(async () => {
120
+ const table = domain.table('workspaces');
121
+ if (table.get(key) === undefined)
122
+ await table.put(key, increment({ entries: {} }, parsed, now));
123
+ else
124
+ await table.update(key, (row) => increment(row, parsed, now));
125
+ })
126
+ .catch(() => {
127
+ this.#diagnostic();
128
+ });
129
+ this.#queues.set(key, job);
130
+ try {
131
+ await job;
132
+ }
133
+ finally {
134
+ if (this.#queues.get(key) === job)
135
+ this.#queues.delete(key);
136
+ }
137
+ }
138
+ async close() {
139
+ this.#closed = true;
140
+ await Promise.all(this.#queues.values());
141
+ await (await this.#ready)?.close();
142
+ }
143
+ }
@@ -1,11 +1,11 @@
1
1
  import { MarivoDatasourceBridge } from './datasource/bridge.ts';
2
2
  import { MarivoHelpBridge } from './disclosure/bridge.ts';
3
3
  import type { MarivoEnvironment } from './environment/binding.ts';
4
- import { MarivoEvidenceBridge } from './evidence/bridge.ts';
4
+ import { MarivoPresentationProjection } from './presentation/projection/index.ts';
5
5
  export interface MarivoBridgeSet {
6
6
  readonly help: MarivoHelpBridge;
7
7
  readonly datasource: MarivoDatasourceBridge;
8
- readonly evidence: MarivoEvidenceBridge;
8
+ readonly presentation: MarivoPresentationProjection;
9
9
  }
10
10
  /** Compose domain adapters without adding domain methods back to the Environment. */
11
11
  export declare function createMarivoBridgeSet(environment: MarivoEnvironment): MarivoBridgeSet;
@@ -0,0 +1,5 @@
1
+ export declare function CreateDatasource({ model, workspaceId, close }: {
2
+ model: any;
3
+ workspaceId: any;
4
+ close: any;
5
+ }): import("react").JSX.Element;
@@ -0,0 +1 @@
1
+ export declare function installCredentials(ctx: any, rpc: any): void;
@@ -0,0 +1,59 @@
1
+ import type { DatasourceAuthoring, DatasourceCreateInput } from '../../datasource/authoring.ts';
2
+ import type { CredentialAction, CredentialContextView, CredentialOperationView, CredentialRequestView } from '../../datasource/service.ts';
3
+ import type { BrowserRpc } from '../semantic-browser/model.ts';
4
+ interface QueryHandle {
5
+ generation: string;
6
+ id: string;
7
+ scope: string;
8
+ }
9
+ export interface ClientOperation {
10
+ handle: QueryHandle;
11
+ name: string;
12
+ workspaceId?: string;
13
+ overviewUpdated?: boolean;
14
+ error?: string;
15
+ operation?: CredentialOperationView;
16
+ }
17
+ export interface CredentialClientState {
18
+ open: boolean;
19
+ workspaceId: string;
20
+ sessionId: string;
21
+ generation: string;
22
+ datasources: CredentialContextView[];
23
+ requests: CredentialRequestView[];
24
+ requestId: string;
25
+ selected: string;
26
+ loading: boolean;
27
+ error: string;
28
+ operations: ClientOperation[];
29
+ outcomes: Record<string, ClientOperation>;
30
+ operation?: CredentialOperationView;
31
+ handle?: QueryHandle;
32
+ }
33
+ export declare function credentialMessage(code: string): string;
34
+ export declare class CredentialClientModel {
35
+ #private;
36
+ constructor(rpc: BrowserRpc, storage?: Pick<Storage, 'getItem' | 'setItem' | 'removeItem'>);
37
+ getSnapshot: () => CredentialClientState;
38
+ subscribe: (listener: () => void) => (() => void);
39
+ show(workspaceId: string): void;
40
+ authoring(workspaceId: string, signal: AbortSignal): Promise<DatasourceAuthoring & {
41
+ generation: string;
42
+ }>;
43
+ createDatasource(workspaceId: string, schema: DatasourceAuthoring & {
44
+ generation: string;
45
+ }, input: DatasourceCreateInput): Promise<void>;
46
+ close(): void;
47
+ openRequest(id: string): void;
48
+ select(token: string): void;
49
+ selectWorkspace(workspaceId: string): Promise<void>;
50
+ session(sessionId: string): void;
51
+ selectOperation(id: string): void;
52
+ start(context: CredentialContextView, action: CredentialAction, changes?: Record<string, string>, reference?: string): Promise<void>;
53
+ recover(): void;
54
+ cancelRequest(id: string): Promise<void>;
55
+ cancelOperation(scope?: string): Promise<void>;
56
+ reset(): void;
57
+ dispose(): void;
58
+ }
59
+ export {};