@v587d/capital-generation 0.0.0-stage → 2.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (117) hide show
  1. package/CHANGELOG.md +794 -0
  2. package/LICENSE +21 -0
  3. package/README.md +116 -2
  4. package/assets/cg-icon.png +0 -0
  5. package/capital-config/client.js +28 -0
  6. package/capital-config/client.src.cjs +388 -0
  7. package/capital-config/index.js +113 -0
  8. package/capital-config/package.json +36 -0
  9. package/capital-watchlist/client.js +182 -0
  10. package/capital-watchlist/client.src.cjs +1229 -0
  11. package/capital-watchlist/index.js +772 -0
  12. package/capital-watchlist/package.json +37 -0
  13. package/chart-ui/README.md +112 -0
  14. package/chart-ui/client.js +16 -0
  15. package/chart-ui/client.src.cjs +126 -0
  16. package/chart-ui/index.js +205 -0
  17. package/chart-ui/package.json +30 -0
  18. package/cordis.patch.yml +72 -0
  19. package/lib/agents/bash-guard.d.ts +55 -0
  20. package/lib/agents/bash-guard.js +162 -0
  21. package/lib/agents/root-tool-policy.d.ts +164 -0
  22. package/lib/agents/root-tool-policy.js +309 -0
  23. package/lib/chart/artifact-ref.d.ts +65 -0
  24. package/lib/chart/artifact-ref.js +107 -0
  25. package/lib/chart/downsample.d.ts +8 -0
  26. package/lib/chart/downsample.js +53 -0
  27. package/lib/chart/errors.d.ts +16 -0
  28. package/lib/chart/errors.js +15 -0
  29. package/lib/chart/events.d.ts +120 -0
  30. package/lib/chart/events.js +471 -0
  31. package/lib/chart/html.d.ts +21 -0
  32. package/lib/chart/html.js +154 -0
  33. package/lib/chart/runtime.d.ts +20 -0
  34. package/lib/chart/runtime.js +270 -0
  35. package/lib/chart/series.d.ts +102 -0
  36. package/lib/chart/series.js +424 -0
  37. package/lib/chart/source-token.d.ts +64 -0
  38. package/lib/chart/source-token.js +217 -0
  39. package/lib/chart/spec.d.ts +58 -0
  40. package/lib/chart/spec.js +265 -0
  41. package/lib/chart/tool.d.ts +86 -0
  42. package/lib/chart/tool.js +288 -0
  43. package/lib/chart/vendor/LICENSE +201 -0
  44. package/lib/chart/vendor/VERSION.json +8 -0
  45. package/lib/chart/vendor/lightweight-charts.standalone.production.js +7 -0
  46. package/lib/data-collector/args.d.ts +38 -0
  47. package/lib/data-collector/args.js +114 -0
  48. package/lib/data-collector/dataset-tools.d.ts +3 -0
  49. package/lib/data-collector/dataset-tools.js +421 -0
  50. package/lib/data-collector/describe.d.ts +99 -0
  51. package/lib/data-collector/describe.js +256 -0
  52. package/lib/data-collector/hub.d.ts +187 -0
  53. package/lib/data-collector/hub.js +469 -0
  54. package/lib/data-collector/query-time.d.ts +58 -0
  55. package/lib/data-collector/query-time.js +101 -0
  56. package/lib/data-collector/query.d.ts +57 -0
  57. package/lib/data-collector/query.js +473 -0
  58. package/lib/data-collector/store.d.ts +499 -0
  59. package/lib/data-collector/store.js +1486 -0
  60. package/lib/data-collector/time-axis.d.ts +169 -0
  61. package/lib/data-collector/time-axis.js +419 -0
  62. package/lib/data-collector/tools.d.ts +32 -0
  63. package/lib/data-collector/tools.js +216 -0
  64. package/lib/index.d.ts +275 -0
  65. package/lib/index.js +458 -0
  66. package/lib/net/eastmoney-client.d.ts +104 -0
  67. package/lib/net/eastmoney-client.js +118 -0
  68. package/lib/net/throttle.d.ts +16 -0
  69. package/lib/net/throttle.js +34 -0
  70. package/lib/ocr/client.d.ts +82 -0
  71. package/lib/ocr/client.js +281 -0
  72. package/lib/ocr/markdown.d.ts +75 -0
  73. package/lib/ocr/markdown.js +209 -0
  74. package/lib/ocr/store.d.ts +119 -0
  75. package/lib/ocr/store.js +226 -0
  76. package/lib/sources/eastmoney-http.d.ts +2 -0
  77. package/lib/sources/eastmoney-http.js +1253 -0
  78. package/lib/sources/fuyao-core.d.ts +52 -0
  79. package/lib/sources/fuyao-core.js +96 -0
  80. package/lib/sources/fuyao-rest.d.ts +14 -0
  81. package/lib/sources/fuyao-rest.js +2057 -0
  82. package/lib/sources/security-code.d.ts +27 -0
  83. package/lib/sources/security-code.js +104 -0
  84. package/lib/sources/security-identity.d.ts +24 -0
  85. package/lib/sources/security-identity.js +89 -0
  86. package/lib/sources/tencent-http.d.ts +2 -0
  87. package/lib/sources/tencent-http.js +1053 -0
  88. package/lib/sources/wind-mcp.d.ts +8 -0
  89. package/lib/sources/wind-mcp.js +593 -0
  90. package/lib/time/tools.d.ts +74 -0
  91. package/lib/time/tools.js +464 -0
  92. package/lib/tool-exec.d.ts +29 -0
  93. package/lib/tool-exec.js +17 -0
  94. package/lib/web-retriever/engines.d.ts +61 -0
  95. package/lib/web-retriever/engines.js +202 -0
  96. package/lib/web-retriever/html-markdown.d.ts +50 -0
  97. package/lib/web-retriever/html-markdown.js +321 -0
  98. package/lib/web-retriever/local-fetch.d.ts +121 -0
  99. package/lib/web-retriever/local-fetch.js +562 -0
  100. package/lib/web-retriever/retriever.d.ts +58 -0
  101. package/lib/web-retriever/retriever.js +94 -0
  102. package/lib/web-retriever/sources.d.ts +235 -0
  103. package/lib/web-retriever/sources.js +961 -0
  104. package/lib/web-retriever/tools.d.ts +50 -0
  105. package/lib/web-retriever/tools.js +1029 -0
  106. package/lib/web-retriever/wind-client.d.ts +69 -0
  107. package/lib/web-retriever/wind-client.js +311 -0
  108. package/locale/en.json +6 -0
  109. package/locale/zh.json +6 -0
  110. package/package.json +76 -4
  111. package/preset/capital-generation/README.md +161 -0
  112. package/preset/capital-generation/agent.patch.yml +509 -0
  113. package/preset/capital-generation/skills/capital-chart-protocol/SKILL.md +137 -0
  114. package/preset/capital-generation/skills/capital-data-protocol/SKILL.md +416 -0
  115. package/preset/capital-generation/skills/capital-orchestration/SKILL.md +174 -0
  116. package/preset/capital-generation/skills/capital-visualization-protocol/SKILL.md +296 -0
  117. package/preset/capital-generation/skills/capital-web-protocol/SKILL.md +240 -0
@@ -0,0 +1,1486 @@
1
+ import { DatasetQueryError, executeJsonRowsQuery } from './query.js';
2
+ import { defaultAxisWindows, isoDateFromEpoch, probeTimeAxis, readAxisDates } from './time-axis.js';
3
+ /**
4
+ * Phase 1/2 workspace-local Dataset store(设计约定见 AGENTS.md「数据布局」)。
5
+ *
6
+ * 原始 Dataset 只由宿主写入,Dataset/Profile 都通过 createIfAbsent 发布。
7
+ * Agent 只能使用 dataset_id 和 opaque artifact_ref,不能传入真实路径。
8
+ */
9
+ export const WORKSPACE_DATA_DIR = 'capital-data';
10
+ export const DATASETS_SUBDIR = `${WORKSPACE_DATA_DIR}/datasets`;
11
+ export const PROFILES_SUBDIR = `${WORKSPACE_DATA_DIR}/profiles`;
12
+ export const RAW_FILE = 'raw.json';
13
+ export const MANIFEST_FILE = 'manifest.json';
14
+ export const PROFILE_FILE = 'profile.json';
15
+ export const DEFAULT_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
16
+ export const MAX_RAW_READ_BYTES = 16 * 1024 * 1024;
17
+ export const MAX_PROFILE_BYTES = 64 * 1024;
18
+ /** 类别事实:最多跟踪多少个不同取值;超过则 distinct_count 记 null 并标记 truncated。 */
19
+ export const MAX_CATEGORY_DISTINCT = 2_000;
20
+ /** `top_values` 只展示前 N 个高频取值,其余由 truncated 显式披露。 */
21
+ export const MAX_CATEGORY_TOP_VALUES = 5;
22
+ const MAX_CATEGORY_VALUE_LENGTH = 64;
23
+ /** 结构摘要:路径条目上限与递归深度上限,保证 profile 体积有界。 */
24
+ export const MAX_STRUCTURE_ENTRIES = 24;
25
+ /**
26
+ * 递归深度上限。行集合里 depth 0 就是列本身,文档里 depth 0 是 `$` 根,
27
+ * 因此文档需要多一层才够到 `abilities[].indicators[]` 这种叶子。
28
+ * 条目上限(24)才是真正的体积闸门,深度只是防止病态深嵌套。
29
+ */
30
+ const MAX_STRUCTURE_DEPTH = 4;
31
+ const MAX_STRUCTURE_FIELDS = 8;
32
+ const MAX_STRUCTURE_SAMPLED_ELEMENTS = 50;
33
+ /**
34
+ * `time_facts.windows` 的条数上限。默认只给 4 个常用窗口(近 1 月/3 月/1 年/年初至今),
35
+ * 上限是防回涨的闸门:窗口是"现成坐标",不是让宿主把任意期间都算一遍。
36
+ */
37
+ const MAX_TIME_WINDOWS = 8;
38
+ /** 推断时间轴时最多检查多少行:命中的列通常在前若干行就出现,不必为一次探针读全量。 */
39
+ const MAX_AXIS_PROBE_ROWS = 500;
40
+ /**
41
+ * 文档型 Dataset(顶层对象、没有行数组,如财务指标 / 回测结果)交给 data_junior 的
42
+ * 内容预算(Unicode 码点数)。取值必须留在 preset 的工具结果剪枝阈值(8192)以内:
43
+ * 超过就会被剪掉中间段,而不是被完整读到。长数组按下面的档位逐级截断,截断了什么
44
+ * 一律写进 `omitted`,不做静默省略。
45
+ */
46
+ export const MAX_DOCUMENT_CHARS = 6_000;
47
+ /**
48
+ * 呈现层(图表)一次性读取行集的上限。超过就拒绝,而不是把几百 MB 拉进宿主内存:
49
+ * 图表是**有界呈现**,不是通用分析通道。
50
+ */
51
+ export const MAX_PRESENTATION_ROWS = 200_000;
52
+ const DOCUMENT_ARRAY_KEEP_STEPS = [8, 3, 1, 0];
53
+ const DOCUMENT_MAX_DEPTH = 6;
54
+ const DOCUMENT_MAX_OMISSIONS = 8;
55
+ export const ARTIFACT_SCHEME = 'workspace://';
56
+ export class DatasetStoreError extends Error {
57
+ code;
58
+ constructor(code, detail) {
59
+ super(`${code}: ${detail}`);
60
+ this.name = 'DatasetStoreError';
61
+ this.code = code;
62
+ }
63
+ }
64
+ function defaultNewId(prefix) {
65
+ const cryptoLike = globalThis.crypto;
66
+ if (cryptoLike?.randomUUID)
67
+ return `${prefix}_${cryptoLike.randomUUID()}`;
68
+ return `${prefix}_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 10)}`;
69
+ }
70
+ export const DATASET_ID_PATTERN = /^[A-Za-z0-9_-]{1,128}$/;
71
+ const PROFILE_ID_PATTERN = /^[A-Za-z0-9_-]{1,128}$/;
72
+ function isAbsolutePath(value) {
73
+ return /^([a-zA-Z]:[\\/]|\/)/.test(value);
74
+ }
75
+ /**
76
+ * 规范化并校验 workspace **相对**路径。
77
+ *
78
+ * 拒绝:空串、绝对路径(含 Windows 盘符)、任何 `..` 段、反斜杠(本仓统一 `/`)。
79
+ * 这里只做形态校验;真正的沙箱归属由 `resolveContained` 证明。两道关卡都要过——
80
+ * 形态校验是为了给出可读错误,归属校验才是安全边界。
81
+ */
82
+ export function normalizeWorkspaceRelativePath(value) {
83
+ if (typeof value !== 'string')
84
+ throw new DatasetStoreError('workspace_path_invalid', 'path must be a string');
85
+ const trimmed = value.trim();
86
+ if (trimmed.length === 0 || trimmed.length > 512)
87
+ throw new DatasetStoreError('workspace_path_invalid', 'path must be 1-512 characters');
88
+ if (trimmed.includes('\\'))
89
+ throw new DatasetStoreError('workspace_path_invalid', 'path must use forward slashes');
90
+ if (isAbsolutePath(trimmed))
91
+ throw new DatasetStoreError('workspace_path_invalid', 'path must be relative to the session workspace');
92
+ const segments = trimmed.split('/').filter((segment) => segment.length > 0);
93
+ if (segments.length === 0)
94
+ throw new DatasetStoreError('workspace_path_invalid', 'path must name a file inside the workspace');
95
+ for (const segment of segments) {
96
+ if (segment === '.' || segment === '..')
97
+ throw new DatasetStoreError('workspace_path_invalid', 'path must not contain "." or ".." segments');
98
+ }
99
+ return segments.join('/');
100
+ }
101
+ function isRecord(value) {
102
+ return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
103
+ }
104
+ function errorCodeOf(error) {
105
+ if (error && typeof error === 'object' && typeof error.code === 'string') {
106
+ return error.code;
107
+ }
108
+ return undefined;
109
+ }
110
+ function errorMessage(error) {
111
+ return error instanceof Error ? error.message : String(error);
112
+ }
113
+ function byteLength(value) {
114
+ return new TextEncoder().encode(value).byteLength;
115
+ }
116
+ function canonicalDatasetRef(datasetId) {
117
+ return `${ARTIFACT_SCHEME}${DATASETS_SUBDIR}/${datasetId}`;
118
+ }
119
+ function canonicalProfileRef(profileId) {
120
+ return `${ARTIFACT_SCHEME}${PROFILES_SUBDIR}/${profileId}`;
121
+ }
122
+ function validNonNegativeInteger(value) {
123
+ return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0;
124
+ }
125
+ export class WorkspaceDatasetStore {
126
+ fs;
127
+ sandboxPolicy;
128
+ retentionMs;
129
+ now;
130
+ newId;
131
+ constructor(options = {}) {
132
+ this.fs = options.fs;
133
+ this.sandboxPolicy = options.sandboxPolicy;
134
+ this.retentionMs = options.retentionMs ?? DEFAULT_RETENTION_MS;
135
+ this.now = options.now ?? (() => Date.now());
136
+ this.newId = options.newId ?? defaultNewId;
137
+ }
138
+ async save(input) {
139
+ const { session, signal } = input;
140
+ const { root, policy } = this.writeContext(session);
141
+ const datasetId = this.newId('ds');
142
+ if (!DATASET_ID_PATTERN.test(datasetId)) {
143
+ throw new DatasetStoreError('dataset_write_failed', `generated dataset_id is invalid: ${datasetId}`);
144
+ }
145
+ const dirPath = `${DATASETS_SUBDIR}/${datasetId}`;
146
+ await this.assertContained(root, dirPath, signal);
147
+ const capturedAt = this.now();
148
+ const manifest = {
149
+ dataset_id: datasetId,
150
+ task_id: input.task_id ?? null,
151
+ session_id: session.id,
152
+ artifact_ref: canonicalDatasetRef(datasetId),
153
+ format: input.format,
154
+ capability: input.capability,
155
+ source_label: input.source_label,
156
+ schema: input.schema,
157
+ row_count: input.row_count,
158
+ captured_at: capturedAt,
159
+ retention_until: capturedAt + this.retentionMs,
160
+ params_digest: input.params_digest,
161
+ };
162
+ const storedManifest = {
163
+ ...manifest,
164
+ session_scope_id: sessionScopeFor(session),
165
+ ...(input.row_key === undefined ? {} : { row_key: input.row_key }),
166
+ };
167
+ let rawContent;
168
+ try {
169
+ rawContent = JSON.stringify(input.data);
170
+ }
171
+ catch (error) {
172
+ throw new DatasetStoreError('dataset_write_failed', `source result is not JSON-serializable: ${errorMessage(error)}`);
173
+ }
174
+ if (typeof rawContent !== 'string') {
175
+ throw new DatasetStoreError('dataset_write_failed', 'source result is not JSON-serializable: JSON.stringify returned no value');
176
+ }
177
+ const rawTarget = await this.resolveContained(`${dirPath}/${RAW_FILE}`, root, signal);
178
+ try {
179
+ await this.fs.writeText(rawTarget, rawContent, { kind: 'createIfAbsent' }, signal, policy);
180
+ }
181
+ catch (error) {
182
+ this.mapWriteError(error, 'writing raw.json');
183
+ }
184
+ const manifestTarget = await this.resolveContained(`${dirPath}/${MANIFEST_FILE}`, root, signal);
185
+ try {
186
+ await this.fs.writeText(manifestTarget, JSON.stringify(storedManifest), { kind: 'createIfAbsent' }, signal, policy);
187
+ }
188
+ catch (error) {
189
+ this.mapWriteError(error, `writing manifest.json (raw.json of ${datasetId} may remain on disk)`);
190
+ }
191
+ return manifest;
192
+ }
193
+ async readRef(datasetId, session, signal) {
194
+ try {
195
+ return await this.requireRef(datasetId, session, signal);
196
+ }
197
+ catch {
198
+ return undefined;
199
+ }
200
+ }
201
+ async listRefs(session, signal) {
202
+ const root = this.readRoot(session);
203
+ if (!root || !this.fs)
204
+ return [];
205
+ try {
206
+ const dirTarget = await this.resolveContained(DATASETS_SUBDIR, root, signal);
207
+ const entries = await this.fs.listDir(dirTarget, signal);
208
+ const refs = [];
209
+ for (const entry of entries) {
210
+ if (entry.type !== 'directory')
211
+ continue;
212
+ const ref = await this.readRef(entry.name, session, signal);
213
+ if (ref)
214
+ refs.push(ref);
215
+ }
216
+ return refs.sort((a, b) => b.captured_at - a.captured_at);
217
+ }
218
+ catch {
219
+ return [];
220
+ }
221
+ }
222
+ async findLatest(input) {
223
+ const refs = await this.listRefs(input.session, input.signal);
224
+ return refs.find((ref) => ref.capability === input.capability && ref.params_digest === input.params_digest);
225
+ }
226
+ async inspectDataset(datasetId, session, signal) {
227
+ const stored = await this.requireStoredRef(datasetId, session, signal);
228
+ const { session_scope_id: _scope, row_key: rowKey, ...ref } = stored;
229
+ try {
230
+ const raw = await this.readRaw(ref, session, signal, rowKey);
231
+ return { ...ref, query_access: { readable: true, shape: raw.kind === 'document' ? 'document' : raw.shape } };
232
+ }
233
+ catch (error) {
234
+ if (error instanceof DatasetStoreError && ['dataset_too_large', 'dataset_not_row_readable'].includes(error.code)) {
235
+ return { ...ref, query_access: { readable: false, shape: 'none', reason: error.code } };
236
+ }
237
+ throw error;
238
+ }
239
+ }
240
+ async profileDataset(input) {
241
+ const stored = await this.requireStoredRef(input.dataset_id, input.session, input.signal);
242
+ const { session_scope_id: _scope, row_key: rowKey, ...ref } = stored;
243
+ const raw = await this.readRaw(ref, input.session, input.signal, rowKey);
244
+ if (raw.kind === 'document') {
245
+ const profile = buildDocumentProfile(raw.value);
246
+ const profileRef = await this.writeProfile({
247
+ session: input.session,
248
+ dataset_id: ref.dataset_id,
249
+ task_id: input.task_id,
250
+ profile,
251
+ signal: input.signal,
252
+ });
253
+ // 结构摘要随 profile 持久化;有界内容只随本次工具结果返回,不进入 profile 产物。
254
+ return { ...profileRef, ...profile, document: boundDocument(raw.value) };
255
+ }
256
+ const profile = buildProfile(raw.rows, input.time_column, input.primary_key, extractRowContracts(ref.schema));
257
+ const profileRef = await this.writeProfile({
258
+ session: input.session,
259
+ dataset_id: ref.dataset_id,
260
+ task_id: input.task_id,
261
+ profile,
262
+ signal: input.signal,
263
+ });
264
+ return { ...profileRef, ...profile };
265
+ }
266
+ async queryDataset(input) {
267
+ const stored = await this.requireStoredRef(input.dataset_id, input.session, input.signal);
268
+ const { session_scope_id: _scope, row_key: rowKey, ...ref } = stored;
269
+ const raw = await this.readRaw(ref, input.session, input.signal, rowKey);
270
+ if (raw.kind !== 'rows')
271
+ throw new DatasetStoreError('dataset_format_unsupported', 'a document Dataset has no rows to query; profile it instead');
272
+ if (input.query.dataset_id !== ref.dataset_id) {
273
+ throw new DatasetQueryError('query_spec_invalid', 'query.dataset_id must match the requested Dataset');
274
+ }
275
+ return executeJsonRowsQuery(raw.rows, ref.schema, input.query);
276
+ }
277
+ /**
278
+ * 时间轴只读探针(`query_dataset` 的日期归一边界与 `resolve_data_time_range` 的数据集模式共用)。
279
+ *
280
+ * 只读前 {@link MAX_AXIS_PROBE_ROWS} 行推断形态与偏移:时间列一定在每行都出现,日线数据
281
+ * 的有序性也让前若干行足以定出粒度;这样这个探针**可以随查询逐次调用**,
282
+ * 代价与"读一份 16MB raw.json 只为一个问题"不同量级。
283
+ */
284
+ async describeTimeAxis(input) {
285
+ const stored = await this.requireStoredRef(input.dataset_id, input.session, input.signal);
286
+ const { session_scope_id: _scope, row_key: rowKey, ...ref } = stored;
287
+ const raw = await this.readRaw(ref, input.session, input.signal, rowKey);
288
+ if (raw.kind !== 'rows' || raw.rows.length === 0)
289
+ return undefined;
290
+ const objectRows = raw.rows.filter(isRecord);
291
+ if (objectRows.length === 0)
292
+ return undefined;
293
+ const columns = [...new Set(objectRows.flatMap((row) => Object.keys(row)))];
294
+ const sample = [];
295
+ const step = Math.max(1, Math.ceil(raw.rows.length / MAX_AXIS_PROBE_ROWS));
296
+ for (let index = 0; index < raw.rows.length && sample.length < MAX_AXIS_PROBE_ROWS; index += step)
297
+ sample.push(raw.rows[index]);
298
+ const column = pickTimeColumn(sample, columns, input.time_column);
299
+ if (column === undefined)
300
+ return undefined;
301
+ const values = sample.map((row) => profileCell(row, column).value);
302
+ const axis = probeTimeAxis(column, values);
303
+ if (axis.value_format === 'unknown')
304
+ return undefined;
305
+ const dates = readAxisDates({ axis, rows: sample });
306
+ if (dates.length === 0)
307
+ return undefined;
308
+ return { dataset_id: ref.dataset_id, columns, axis, dates };
309
+ }
310
+ async writeProfile(input) {
311
+ const { session, signal } = input;
312
+ const ref = await this.requireRef(input.dataset_id, session, signal);
313
+ const profile = validateProfile(input.profile);
314
+ const encodedProfile = JSON.stringify(profile);
315
+ if (byteLength(encodedProfile) > MAX_PROFILE_BYTES)
316
+ throw new DatasetStoreError('profile_too_large', `profile exceeds ${MAX_PROFILE_BYTES} bytes`);
317
+ const { root, policy } = this.writeContext(session);
318
+ const profileId = this.newId('p');
319
+ if (!PROFILE_ID_PATTERN.test(profileId))
320
+ throw new DatasetStoreError('profile_invalid', `generated profile_id is invalid: ${profileId}`);
321
+ const createdAt = this.now();
322
+ const profileRef = {
323
+ profile_id: profileId,
324
+ dataset_id: ref.dataset_id,
325
+ task_id: input.task_id ?? null,
326
+ session_id: session.id,
327
+ artifact_ref: canonicalProfileRef(profileId),
328
+ created_at: createdAt,
329
+ retention_until: Math.min(ref.retention_until, createdAt + this.retentionMs),
330
+ };
331
+ const target = await this.resolveContained(`${PROFILES_SUBDIR}/${profileId}/${PROFILE_FILE}`, root, signal);
332
+ try {
333
+ await this.fs.writeText(target, JSON.stringify({ ref: profileRef, profile }), { kind: 'createIfAbsent' }, signal, policy);
334
+ }
335
+ catch (error) {
336
+ this.mapWriteError(error, `writing profile.json (${profileId})`);
337
+ }
338
+ return profileRef;
339
+ }
340
+ /**
341
+ * 宿主内部**呈现层**读取有界行集(图表用)。
342
+ *
343
+ * 契约(不可违反):
344
+ * - 返回值**不得**进入任何 Agent 消息、工具结果或模型上下文;调用方只能在宿主进程内
345
+ * 把它变成有界可视化载荷(见 src/chart/series.ts)。
346
+ * - 这是唯一一处把原始 rows 交给 store 之外代码的入口,因此命名、文档与调用点都必须是
347
+ * "呈现"语义。需要数值结论一律走 profile_dataset / query_dataset。
348
+ * - session 作用域与 profile/query 同源(`sessionScopeFor` 取父会话),所以主 Agent
349
+ * 用自己会话即可读到自己子 Agent 采到的 Dataset。
350
+ */
351
+ async readPresentationRows(input) {
352
+ const stored = await this.requireStoredRef(input.dataset_id, input.session, input.signal);
353
+ const { session_scope_id: _scope, row_key: rowKey, ...ref } = stored;
354
+ const raw = await this.readRaw(ref, input.session, input.signal, rowKey);
355
+ if (raw.kind !== 'rows') {
356
+ throw new DatasetStoreError('dataset_format_unsupported', 'a document Dataset has no rows to present; profile it instead');
357
+ }
358
+ const maxRows = input.maxRows ?? MAX_PRESENTATION_ROWS;
359
+ if (raw.rows.length > maxRows) {
360
+ throw new DatasetStoreError('dataset_too_large', `Dataset has ${raw.rows.length} rows; presentation reads are capped at ${maxRows}`);
361
+ }
362
+ return { ref, rows: raw.rows };
363
+ }
364
+ /**
365
+ * 宿主内部读取 workspace 内的**文本**文件(图表 spec、OCR 产物、用户本地数据)。
366
+ *
367
+ * 这是 workspace 读取的**唯一**实现:`readWorkspaceJson` / `readWorkspaceBytes` 都从
368
+ * `workspaceFileTarget` 取归属与体积闸门,不各自再写一遍路径校验(§9.7——同一个事实
369
+ * 有两份实现就一定会漂移)。
370
+ */
371
+ async readWorkspaceText(input) {
372
+ const maxBytes = input.maxBytes ?? MAX_RAW_READ_BYTES;
373
+ const { relative, target } = await this.workspaceFileTarget(input.session, input.path, input.signal, maxBytes);
374
+ let text;
375
+ try {
376
+ text = await this.fs.readText(target, input.signal);
377
+ }
378
+ catch (error) {
379
+ if (errorCodeOf(error) === 'FS_NOT_FOUND')
380
+ throw new DatasetStoreError('dataset_not_found', `${relative} was not found`);
381
+ throw new DatasetStoreError('workspace_file_invalid', `${relative} could not be read`);
382
+ }
383
+ if (byteLength(text) > maxBytes) {
384
+ throw new DatasetStoreError('dataset_too_large', `${relative} exceeds ${maxBytes} bytes`);
385
+ }
386
+ return text;
387
+ }
388
+ /**
389
+ * 宿主内部读取 workspace 内的**原始字节**(本地 PDF / 图片上传给外部解析服务的入口)。
390
+ *
391
+ * 上限必须显式给出:这道缝后面的调用方要把字节整个装进内存,没有默认值可兜。
392
+ * `fs.readBytes` 缺席(载体未挂载该能力)时**响亮失败**,不回退成 `readText`——
393
+ * 二进制走文本读取会得到被解码污染的字节。
394
+ */
395
+ async readWorkspaceBytes(input) {
396
+ const { relative, target } = await this.workspaceFileTarget(input.session, input.path, input.signal, input.maxBytes);
397
+ if (typeof this.fs?.readBytes !== 'function') {
398
+ throw new DatasetStoreError('filesystem_unavailable', 'the mounted filesystem cannot read raw bytes');
399
+ }
400
+ try {
401
+ return await this.fs.readBytes(target, input.signal, input.maxBytes);
402
+ }
403
+ catch (error) {
404
+ const code = errorCodeOf(error);
405
+ if (code === 'FS_NOT_FOUND')
406
+ throw new DatasetStoreError('dataset_not_found', `${relative} was not found`);
407
+ if (code === 'FS_TOO_LARGE')
408
+ throw new DatasetStoreError('dataset_too_large', `${relative} exceeds ${input.maxBytes} bytes`);
409
+ throw new DatasetStoreError('workspace_file_invalid', `${relative} could not be read`);
410
+ }
411
+ }
412
+ async workspaceFileTarget(session, path, signal, maxBytes) {
413
+ const root = this.readRoot(session);
414
+ if (!root || !this.fs)
415
+ throw new DatasetStoreError('filesystem_unavailable', 'workspace filesystem is unavailable');
416
+ const relative = normalizeWorkspaceRelativePath(path);
417
+ const target = await this.resolveContained(relative, root, signal);
418
+ const info = this.fs.stat ? await this.fs.stat(target, signal) : undefined;
419
+ if (info && info.type !== 'file')
420
+ throw new DatasetStoreError('workspace_file_invalid', `${relative} is not a regular file`);
421
+ if (typeof info?.size === 'number' && info.size > maxBytes) {
422
+ throw new DatasetStoreError('dataset_too_large', `${relative} exceeds ${maxBytes} bytes`);
423
+ }
424
+ return { relative, target };
425
+ }
426
+ /** 宿主内部呈现层读取 workspace 内的 JSON 文件(用户本地数据直接可视化的入口)。 */
427
+ async readWorkspaceJson(input) {
428
+ const text = await this.readWorkspaceText(input);
429
+ try {
430
+ return JSON.parse(text);
431
+ }
432
+ catch {
433
+ throw new DatasetStoreError('workspace_file_invalid', `${input.path} is not valid JSON`);
434
+ }
435
+ }
436
+ /**
437
+ * 宿主内部**产物层**把一组文本文件写进 workspace 的一个子目录(图表与 OCR 产物用)。
438
+ *
439
+ * 走与 Dataset 落盘完全相同的沙箱策略与归属校验;默认 `createIfAbsent`,保证不会静默
440
+ * 覆盖已有产物(图表 id 唯一,重复即 bug,应当响亮失败)。
441
+ *
442
+ * `overwrite: true` 是给**内容寻址产物**重跑用的(OCR:上次在两个文件之间被中断,
443
+ * 半件产物必须能被下一次解析自愈)。它不是"跳过幂等"的口子:幂等判断在调用方,
444
+ * 只有已经决定重新生成时才传。
445
+ *
446
+ * 返回值同时给出**相对路径**(可以进 Agent 消息、可以 present 给用户)与**绝对路径**
447
+ * (只供宿主内部使用,例如登记给 host 平面的取数路由)。这个区分是本仓的数据纪律:
448
+ * 绝对路径不出宿主进程。
449
+ */
450
+ async writeWorkspaceFiles(input) {
451
+ const { session, signal } = input;
452
+ const { root, policy } = this.writeContext(session);
453
+ const dir = normalizeWorkspaceRelativePath(input.dir);
454
+ await this.assertContained(root, dir, signal);
455
+ const written = [];
456
+ for (const file of input.files) {
457
+ if (!/^[A-Za-z0-9._-]{1,64}$/.test(file.name)) {
458
+ throw new DatasetStoreError('workspace_path_invalid', `artifact name is invalid: ${file.name}`);
459
+ }
460
+ const relative = `${dir}/${file.name}`;
461
+ const target = await this.resolveContained(relative, root, signal);
462
+ try {
463
+ await this.fs.writeText(target, file.content, input.overwrite ? undefined : { kind: 'createIfAbsent' }, signal, policy);
464
+ }
465
+ catch (error) {
466
+ this.mapWriteError(error, `writing ${relative}`);
467
+ }
468
+ written.push({ name: file.name, path: relative, absolutePath: target.displayPath });
469
+ }
470
+ return written;
471
+ }
472
+ async requireRef(datasetId, session, signal) {
473
+ const stored = await this.requireStoredRef(datasetId, session, signal);
474
+ const { session_scope_id: _scope, row_key: _rowKey, ...ref } = stored;
475
+ return ref;
476
+ }
477
+ async requireStoredRef(datasetId, session, signal) {
478
+ if (!DATASET_ID_PATTERN.test(datasetId))
479
+ throw new DatasetStoreError('dataset_id_invalid', 'dataset_id is invalid');
480
+ if (!session || typeof session.id !== 'string' || session.id.length === 0)
481
+ throw new DatasetStoreError('session_unavailable', 'the calling agent session has no id');
482
+ const root = this.readRoot(session);
483
+ if (!root || !this.fs)
484
+ throw new DatasetStoreError('filesystem_unavailable', 'workspace filesystem is unavailable');
485
+ let text;
486
+ try {
487
+ const target = await this.resolveContained(`${DATASETS_SUBDIR}/${datasetId}/${MANIFEST_FILE}`, root, signal);
488
+ text = await this.fs.readText(target, signal);
489
+ }
490
+ catch (error) {
491
+ if (errorCodeOf(error) === 'FS_NOT_FOUND')
492
+ throw new DatasetStoreError('dataset_not_found', `Dataset ${datasetId} was not found`);
493
+ if (error instanceof DatasetStoreError)
494
+ throw error;
495
+ throw new DatasetStoreError('dataset_manifest_invalid', 'Dataset manifest could not be read');
496
+ }
497
+ let parsed;
498
+ try {
499
+ parsed = JSON.parse(text);
500
+ }
501
+ catch {
502
+ throw new DatasetStoreError('dataset_manifest_invalid', 'Dataset manifest is not valid JSON');
503
+ }
504
+ const stored = validateStoredDatasetManifest(parsed, datasetId);
505
+ if (!stored)
506
+ throw new DatasetStoreError('dataset_manifest_invalid', 'Dataset manifest fields are invalid');
507
+ if (stored.session_scope_id !== sessionScopeFor(session))
508
+ throw new DatasetStoreError('dataset_session_mismatch', 'Dataset belongs to another session scope');
509
+ if (stored.retention_until <= this.now())
510
+ throw new DatasetStoreError('dataset_expired', `Dataset ${datasetId} has expired`);
511
+ return stored;
512
+ }
513
+ /**
514
+ * 读取原始 Dataset。返回两种形状之一:
515
+ * - `rows`:顶层数组或 manifest 声明的行数组键,可 profile、可 query;
516
+ * - `document`:顶层对象且没有行数组(财务指标、回测结果这类),可 profile
517
+ * (宿主算结构摘要 + 有界内容),不可 query。
518
+ * 判定放在读取时而不是复用 manifest 的 `format`:`format` 只记录写入时的粗略
519
+ * 形状,真正的形状以文件内容为准。
520
+ */
521
+ async readRaw(ref, session, signal, rowKey) {
522
+ const root = this.readRoot(session);
523
+ if (!root || !this.fs)
524
+ throw new DatasetStoreError('filesystem_unavailable', 'workspace filesystem is unavailable');
525
+ const target = await this.resolveContained(`${DATASETS_SUBDIR}/${ref.dataset_id}/${RAW_FILE}`, root, signal);
526
+ const info = this.fs.stat ? await this.fs.stat(target, signal) : undefined;
527
+ if (info && info.type !== 'file')
528
+ throw new DatasetStoreError('dataset_not_row_readable', 'raw Dataset object is not a regular file');
529
+ if (typeof info?.size === 'number' && info.size > MAX_RAW_READ_BYTES)
530
+ throw new DatasetStoreError('dataset_too_large', `raw Dataset exceeds ${MAX_RAW_READ_BYTES} bytes`);
531
+ let text;
532
+ try {
533
+ text = await this.fs.readText(target, signal);
534
+ }
535
+ catch {
536
+ throw new DatasetStoreError('dataset_not_row_readable', 'raw Dataset could not be read');
537
+ }
538
+ if (byteLength(text) > MAX_RAW_READ_BYTES)
539
+ throw new DatasetStoreError('dataset_too_large', `raw Dataset exceeds ${MAX_RAW_READ_BYTES} bytes`);
540
+ let value;
541
+ try {
542
+ value = JSON.parse(text);
543
+ }
544
+ catch {
545
+ throw new DatasetStoreError('dataset_not_row_readable', 'raw Dataset is not valid JSON');
546
+ }
547
+ if (Array.isArray(value))
548
+ return { kind: 'rows', rows: value, shape: 'array' };
549
+ if (isRecord(value)) {
550
+ const rows = rowsAtPath(value, rowKey ?? 'item');
551
+ if (rows !== undefined)
552
+ return { kind: 'rows', rows, shape: 'envelope_item' };
553
+ }
554
+ if (isRecord(value))
555
+ return { kind: 'document', value };
556
+ throw new DatasetStoreError('dataset_not_row_readable', 'raw Dataset does not contain a supported row array or document object');
557
+ }
558
+ async resolveContained(relativePath, root, signal) {
559
+ if (!this.fs)
560
+ throw new DatasetStoreError('filesystem_unavailable', 'DSH filesystem service is not mounted');
561
+ const target = await this.fs.resolve(relativePath, { cwd: root, signal });
562
+ const rootTarget = await this.fs.resolve('.', { cwd: root, signal });
563
+ if (typeof this.fs.contains !== 'function' || !this.fs.contains(rootTarget, target)) {
564
+ throw new DatasetStoreError('workspace_not_writable', 'filesystem cannot prove workspace containment');
565
+ }
566
+ return target;
567
+ }
568
+ async assertContained(root, relativePath, signal) {
569
+ await this.resolveContained(relativePath, root, signal);
570
+ }
571
+ writeContext(session) {
572
+ if (!this.fs)
573
+ throw new DatasetStoreError('filesystem_unavailable', 'DSH filesystem is not mounted; cannot persist a Dataset');
574
+ if (typeof this.sandboxPolicy?.resolve !== 'function')
575
+ throw new DatasetStoreError('sandbox_policy_unavailable', 'DSH sandbox policy is not mounted');
576
+ if (!session || typeof session.id !== 'string' || session.id.length === 0)
577
+ throw new DatasetStoreError('session_unavailable', 'the calling agent session has no id');
578
+ const cwd = session.header?.cwd;
579
+ if (typeof cwd !== 'string' || !isAbsolutePath(cwd))
580
+ throw new DatasetStoreError('session_cwd_unavailable', 'calling agent workspace cwd is unavailable or not absolute');
581
+ const policy = this.sandboxPolicy.resolve({ session });
582
+ if (policy?.mode !== 'workspace-write' && policy?.mode !== 'danger-full-access') {
583
+ throw new DatasetStoreError('workspace_not_writable', `current workspace permission is ${policy?.mode ?? 'unknown'}`);
584
+ }
585
+ const root = typeof policy.workspaceRoot === 'string' && policy.workspaceRoot.length > 0 ? policy.workspaceRoot : cwd;
586
+ if (!isAbsolutePath(root))
587
+ throw new DatasetStoreError('workspace_not_writable', 'resolved workspace root is not absolute');
588
+ return { root, policy };
589
+ }
590
+ readRoot(session) {
591
+ if (!this.fs)
592
+ return undefined;
593
+ const cwd = session?.header?.cwd;
594
+ const policyRoot = typeof this.sandboxPolicy?.resolve === 'function' ? this.sandboxPolicy.resolve({ session })?.workspaceRoot : undefined;
595
+ const root = typeof policyRoot === 'string' && policyRoot.length > 0 ? policyRoot : cwd;
596
+ if (typeof root !== 'string' || !isAbsolutePath(root))
597
+ return undefined;
598
+ return root;
599
+ }
600
+ mapWriteError(error, action) {
601
+ const code = errorCodeOf(error);
602
+ if (code === 'FS_SANDBOX_DENIED' || code === 'FS_PERMISSION_DENIED') {
603
+ throw new DatasetStoreError('workspace_not_writable', `${action} denied by the DSH file sandbox (${code})`);
604
+ }
605
+ throw new DatasetStoreError('dataset_write_failed', `${action} failed${code ? ` (${code})` : ''}`);
606
+ }
607
+ }
608
+ function rowsAtPath(value, path) {
609
+ const segments = path.split('.').filter(Boolean);
610
+ const visit = (current, index) => {
611
+ if (index === segments.length)
612
+ return Array.isArray(current) ? current : undefined;
613
+ if (!current || typeof current !== 'object' || Array.isArray(current))
614
+ return undefined;
615
+ const segment = segments[index];
616
+ if (segment.endsWith('[]')) {
617
+ const child = current[segment.slice(0, -2)];
618
+ if (!Array.isArray(child))
619
+ return undefined;
620
+ const merged = [];
621
+ for (const item of child) {
622
+ const rows = visit(item, index + 1);
623
+ if (rows === undefined)
624
+ return undefined;
625
+ merged.push(...rows);
626
+ }
627
+ return merged;
628
+ }
629
+ return visit(current[segment], index + 1);
630
+ };
631
+ return visit(value, 0);
632
+ }
633
+ function sessionScopeFor(session) {
634
+ const parentSession = session.header?.parentSession;
635
+ return typeof parentSession === 'string' && parentSession.length > 0 ? parentSession : session.id;
636
+ }
637
+ function buildProfile(rows, requestedTimeColumn, primaryKey, contracts = {}) {
638
+ const objectRows = rows.filter(isRecord);
639
+ const observedColumns = objectRows.length > 0
640
+ ? [...new Set(objectRows.flatMap((row) => Object.keys(row)))]
641
+ : rows.length > 0 ? ['value'] : [];
642
+ const columns = [...new Set([...observedColumns, ...Object.keys(contracts)])];
643
+ const warnings = [];
644
+ if (objectRows.length !== rows.length)
645
+ warnings.push('some rows are not objects; object-column checks are partial');
646
+ let missingValues = 0;
647
+ for (const row of rows) {
648
+ if (!isRecord(row)) {
649
+ if (columns.includes('value') && row === null)
650
+ missingValues += 1;
651
+ continue;
652
+ }
653
+ for (const column of observedColumns)
654
+ if (!(column in row) || row[column] === null || row[column] === undefined)
655
+ missingValues += 1;
656
+ }
657
+ const seen = new Set();
658
+ let duplicateRows = 0;
659
+ for (const row of rows) {
660
+ const key = primaryKey && isRecord(row)
661
+ ? stableProfileValue(row[primaryKey])
662
+ : stableProfileValue(row);
663
+ if (seen.has(key))
664
+ duplicateRows += 1;
665
+ else
666
+ seen.add(key);
667
+ }
668
+ const timeColumn = pickTimeColumn(rows, columns, requestedTimeColumn);
669
+ let timeOrdered = null;
670
+ if (timeColumn) {
671
+ const values = rows.map((row) => isRecord(row) ? row[timeColumn] : undefined);
672
+ if (values.every((value) => value !== undefined && value !== null)) {
673
+ timeOrdered = true;
674
+ for (let index = 1; index < values.length; index += 1) {
675
+ if (compareProfileValues(values[index - 1], values[index]) > 0) {
676
+ timeOrdered = false;
677
+ break;
678
+ }
679
+ }
680
+ }
681
+ else {
682
+ warnings.push(`time column ${timeColumn} contains missing values`);
683
+ }
684
+ }
685
+ const statistics = {};
686
+ const categories = {};
687
+ const structure = {};
688
+ const categoryTally = new Map();
689
+ const structureEntries = new Map();
690
+ let structureTruncated = false;
691
+ const schema = {};
692
+ const violations = [];
693
+ for (const column of columns) {
694
+ const contract = contracts[column];
695
+ const observedTypes = {};
696
+ let missingCount = 0;
697
+ let nullCount = 0;
698
+ let nonNullCount = 0;
699
+ let invalidCount = 0;
700
+ let numericStringCount = 0;
701
+ for (const row of rows) {
702
+ const cell = profileCell(row, column);
703
+ if (!cell.present || cell.value === undefined) {
704
+ missingCount += 1;
705
+ incrementTypeCount(observedTypes, 'missing');
706
+ continue;
707
+ }
708
+ const type = profileObservedType(cell.value);
709
+ incrementTypeCount(observedTypes, type);
710
+ if (cell.value === null) {
711
+ nullCount += 1;
712
+ continue;
713
+ }
714
+ nonNullCount += 1;
715
+ if (contract && isNumericContract(contract.type) && typeof cell.value === 'string' && isFiniteNumericString(cell.value)) {
716
+ numericStringCount += 1;
717
+ }
718
+ if (contract && !matchesProfileContract(cell.value, contract))
719
+ invalidCount += 1;
720
+ if (typeof cell.value === 'string')
721
+ tallyCategoryValue(categoryTally, column, cell.value);
722
+ if (Array.isArray(cell.value) || isRecord(cell.value)) {
723
+ if (!structureEntries.has(column) && structureEntries.size >= MAX_STRUCTURE_ENTRIES)
724
+ structureTruncated = true;
725
+ else
726
+ collectStructure(cell.value, column, 0, structureEntries);
727
+ }
728
+ }
729
+ const inferredType = inferProfileType(observedTypes);
730
+ const columnProfile = {
731
+ contract_type: contract?.type ?? null,
732
+ inferred_type: inferredType,
733
+ observed_types: observedTypes,
734
+ missing_count: missingCount,
735
+ null_count: nullCount,
736
+ non_null_count: nonNullCount,
737
+ invalid_count: invalidCount,
738
+ nullable: nullCount > 0,
739
+ ...(contract?.nullable === undefined ? {} : { contract_nullable: contract.nullable }),
740
+ ...(contract?.required === undefined ? {} : { required: contract.required }),
741
+ ...(contract?.allowNumericString === undefined ? {} : { allow_numeric_string: contract.allowNumericString }),
742
+ ...(numericStringCount > 0 ? { numeric_string_count: numericStringCount } : {}),
743
+ };
744
+ schema[column] = columnProfile;
745
+ if (contract?.required && missingCount > 0) {
746
+ violations.push({ column, rule: 'required', expected: true, observed: 'missing', invalid_count: missingCount, severity: 'error' });
747
+ }
748
+ if (contract?.nullable === false && nullCount > 0) {
749
+ violations.push({ column, rule: 'nullable', expected: false, observed: 'null', invalid_count: nullCount, severity: 'error' });
750
+ }
751
+ if (invalidCount > 0) {
752
+ violations.push({ column, rule: 'type', expected: contract?.type ?? 'unknown', observed: inferredType, invalid_count: invalidCount, severity: 'error' });
753
+ }
754
+ const values = rows
755
+ .map((row) => profileCell(row, column))
756
+ .filter((cell) => cell.present && typeof cell.value === 'number' && Number.isFinite(cell.value))
757
+ .map((cell) => cell.value);
758
+ if (values.length > 0) {
759
+ const sorted = [...values].sort((a, b) => a - b);
760
+ const total = values.reduce((sum, value) => sum + value, 0);
761
+ statistics[column] = {
762
+ count: values.length,
763
+ sum: Number.isFinite(total) ? total : null,
764
+ min: sorted[0],
765
+ max: sorted[sorted.length - 1],
766
+ mean: total / values.length,
767
+ p25: profileQuantile(sorted, 0.25),
768
+ p50: profileQuantile(sorted, 0.5),
769
+ p75: profileQuantile(sorted, 0.75),
770
+ };
771
+ }
772
+ if (inferredType === 'string') {
773
+ const tally = categoryTally.get(column);
774
+ const distinctCount = tally === undefined || tally.overflowed ? null : tally.counts.size;
775
+ const top = [...(tally?.counts ?? new Map()).entries()]
776
+ .sort((left, right) => right[1] - left[1] || (left[0] < right[0] ? -1 : left[0] > right[0] ? 1 : 0))
777
+ .slice(0, MAX_CATEGORY_TOP_VALUES)
778
+ .map(([value, count]) => ({ value, count }));
779
+ categories[column] = {
780
+ distinct_count: distinctCount,
781
+ top_values: top,
782
+ truncated: distinctCount === null || distinctCount > top.length,
783
+ };
784
+ }
785
+ }
786
+ for (const [path, accumulator] of structureEntries)
787
+ structure[path] = finishStructure(accumulator);
788
+ if (structureTruncated)
789
+ warnings.push(`structure digest truncated at ${MAX_STRUCTURE_ENTRIES} paths`);
790
+ if ([...structureEntries.values()].some((accumulator) => accumulator.sampled))
791
+ warnings.push(`structure digest sampled after ${MAX_STRUCTURE_SAMPLED_ELEMENTS} elements; sampled_elements is incomplete`);
792
+ const timeFacts = timeColumn === undefined
793
+ ? undefined
794
+ : timeFactsFor(rows, timeColumn, timeOrdered, new Set(Object.keys(statistics)));
795
+ return {
796
+ row_count: rows.length,
797
+ columns,
798
+ quality: { missing_values: missingValues, duplicate_rows: duplicateRows, time_ordered: timeOrdered },
799
+ ...(Object.keys(statistics).length > 0 ? { statistics } : {}),
800
+ ...(Object.keys(categories).length > 0 ? { categories } : {}),
801
+ ...(timeFacts === undefined ? {} : { time_facts: timeFacts }),
802
+ ...(Object.keys(structure).length > 0 ? { structure } : {}),
803
+ schema,
804
+ validation: { status: violations.length > 0 ? 'fail' : 'pass', violations },
805
+ warnings,
806
+ };
807
+ }
808
+ /**
809
+ * 挑时间列:显式指定优先;否则按列名候选,并要求该列的非空值全是数字或日期样式
810
+ * 字符串。只按列名取第一个匹配列会把 `report_type`(季度枚举)误当时间轴,
811
+ * 于是首末值与覆盖范围全是错的口径。
812
+ */
813
+ function pickTimeColumn(rows, columns, requested) {
814
+ if (requested !== undefined)
815
+ return columns.includes(requested) ? requested : requested;
816
+ for (const column of columns) {
817
+ if (!/date|time|timestamp|_dt$|report$/i.test(column))
818
+ continue;
819
+ const values = rows
820
+ .map((row) => profileCell(row, column))
821
+ .filter((cell) => cell.present && cell.value !== null && cell.value !== undefined)
822
+ .map((cell) => cell.value);
823
+ if (values.length === 0)
824
+ continue;
825
+ if (values.every((value) => typeof value === 'number' && Number.isFinite(value)))
826
+ return column;
827
+ if (values.every((value) => typeof value === 'string' && DATE_LIKE_PATTERN.test(value.trim())))
828
+ return column;
829
+ }
830
+ return undefined;
831
+ }
832
+ const DATE_LIKE_PATTERN = /^\d{4}(-\d{1,2}(-\d{1,2})?)?$/;
833
+ /**
834
+ * 时间事实:覆盖范围 + 首行/末行的「时间列与数值列」取值(文件顺序)。
835
+ *
836
+ * 只报 min/max 时,模型会把区间极值讲成走势(实测样本里出现过「从 74 涨到 105
837
+ * 以上随后可能回落」这种没有依据的叙述)。首末行给出的是真实首末取值,
838
+ * 区间变化因此可追溯;`ordered_ascending` 说明文件顺序,避免把末行当成最新。
839
+ */
840
+ /**
841
+ * 时间事实的**唯一实现**(profile 与时间轴探针共用):
842
+ * 覆盖范围(原值 + ISO)、时间轴语义、常用窗口、首末行取值。
843
+ */
844
+ function timeFactsFor(rows, timeColumn, orderedAscending, numericColumns) {
845
+ const project = (row) => {
846
+ const projected = {};
847
+ if (!isRecord(row))
848
+ return projected;
849
+ for (const column of [timeColumn, ...numericColumns]) {
850
+ if (column in projected)
851
+ continue;
852
+ const cell = profileCell(row, column);
853
+ if (cell.present && cell.value !== undefined)
854
+ projected[column] = cell.value;
855
+ }
856
+ return projected;
857
+ };
858
+ const timeValues = rows.map((row) => profileCell(row, timeColumn).value).filter((value) => value !== null && value !== undefined);
859
+ let coveredFrom = null;
860
+ let coveredTo = null;
861
+ for (const value of timeValues) {
862
+ if (coveredFrom === null || compareProfileValues(value, coveredFrom) < 0)
863
+ coveredFrom = value;
864
+ if (coveredTo === null || compareProfileValues(value, coveredTo) > 0)
865
+ coveredTo = value;
866
+ }
867
+ const axis = probeTimeAxis(timeColumn, timeValues);
868
+ const dates = readAxisDates({ axis, rows });
869
+ const coveredFromIso = typeof coveredFrom === 'number' && axis.value_format === 'epoch_ms'
870
+ ? isoDateFromEpoch(coveredFrom, axis.time_zone ?? 'UTC')
871
+ : undefined;
872
+ const coveredToIso = typeof coveredTo === 'number' && axis.value_format === 'epoch_ms'
873
+ ? isoDateFromEpoch(coveredTo, axis.time_zone ?? 'UTC')
874
+ : undefined;
875
+ const windows = axis.value_format === 'unknown' ? [] : defaultAxisWindows({ axis, dates });
876
+ return {
877
+ time_column: timeColumn,
878
+ ordered_ascending: orderedAscending,
879
+ covered_from: coveredFrom,
880
+ covered_to: coveredTo,
881
+ ...(coveredFromIso === undefined ? {} : { covered_from_iso: coveredFromIso }),
882
+ ...(coveredToIso === undefined ? {} : { covered_to_iso: coveredToIso }),
883
+ axis,
884
+ ...(windows.length === 0 ? {} : { windows }),
885
+ first: project(rows[0]),
886
+ last: project(rows[rows.length - 1]),
887
+ };
888
+ }
889
+ function structureAccumulator(entries, path, type) {
890
+ const existing = entries.get(path);
891
+ if (existing !== undefined)
892
+ return existing;
893
+ if (entries.size >= MAX_STRUCTURE_ENTRIES)
894
+ return undefined;
895
+ const created = { type, occurrences: 0, totalElements: 0, emptyCount: 0, minLength: 0, maxLength: 0, sampledElements: 0, sampled: false, fields: new Map() };
896
+ entries.set(path, created);
897
+ return created;
898
+ }
899
+ /**
900
+ * 把单元格里的数组/对象展开成有界路径摘要(深度 ≤3、条目 ≤24)。
901
+ *
902
+ * 路径约定:对象字段用 `.name`,数组元素用 `[]`,例如 QDII 额度的三层嵌套会得到
903
+ * `sub_tab`(数组,共 3 项)→ `sub_tab[].fund_list`(数组,共 40 项)。
904
+ * 未抽样时用 `total_elements` 表示完整数量;超过 50 项时改用 `sampled_elements`,明确只统计了实际检查到的部分,
905
+ * 而不是让模型面对一句「sub_tab 的类型是 object」。
906
+ */
907
+ function collectStructure(value, path, depth, entries, sampled = false) {
908
+ if (depth > MAX_STRUCTURE_DEPTH)
909
+ return;
910
+ if (Array.isArray(value)) {
911
+ const accumulator = structureAccumulator(entries, path, 'array');
912
+ if (accumulator === undefined)
913
+ return;
914
+ accumulator.occurrences += 1;
915
+ if (sampled || value.length > MAX_STRUCTURE_SAMPLED_ELEMENTS) {
916
+ accumulator.sampled = true;
917
+ accumulator.sampledElements += Math.min(value.length, MAX_STRUCTURE_SAMPLED_ELEMENTS);
918
+ }
919
+ else {
920
+ accumulator.totalElements += value.length;
921
+ }
922
+ if (value.length === 0)
923
+ accumulator.emptyCount += 1;
924
+ if (accumulator.occurrences === 1) {
925
+ accumulator.minLength = value.length;
926
+ accumulator.maxLength = value.length;
927
+ }
928
+ else {
929
+ accumulator.minLength = Math.min(accumulator.minLength, value.length);
930
+ accumulator.maxLength = Math.max(accumulator.maxLength, value.length);
931
+ }
932
+ const sampledChildren = sampled || value.length > MAX_STRUCTURE_SAMPLED_ELEMENTS;
933
+ for (const element of value.slice(0, MAX_STRUCTURE_SAMPLED_ELEMENTS)) {
934
+ if (Array.isArray(element) || isRecord(element))
935
+ collectStructure(element, `${path}[]`, depth + 1, entries, sampledChildren);
936
+ }
937
+ return;
938
+ }
939
+ if (!isRecord(value))
940
+ return;
941
+ const accumulator = structureAccumulator(entries, path, 'object');
942
+ if (accumulator === undefined)
943
+ return;
944
+ accumulator.occurrences += 1;
945
+ if (sampled) {
946
+ accumulator.sampled = true;
947
+ accumulator.sampledElements += 1;
948
+ }
949
+ for (const [key, child] of Object.entries(value)) {
950
+ if (accumulator.fields.size < MAX_STRUCTURE_FIELDS) {
951
+ const type = child === null || child === undefined ? 'null' : Array.isArray(child) ? 'array' : isRecord(child) ? 'object' : typeof child;
952
+ if (!accumulator.fields.has(key))
953
+ accumulator.fields.set(key, type);
954
+ }
955
+ if (Array.isArray(child) || isRecord(child))
956
+ collectStructure(child, `${path}.${key}`, depth + 1, entries, sampled);
957
+ }
958
+ }
959
+ function finishStructure(accumulator) {
960
+ return {
961
+ type: accumulator.type,
962
+ occurrences: accumulator.occurrences,
963
+ ...(accumulator.sampled ? { sampled_elements: accumulator.sampledElements } : { total_elements: accumulator.totalElements }),
964
+ empty_count: accumulator.emptyCount,
965
+ min_length: accumulator.minLength,
966
+ max_length: accumulator.maxLength,
967
+ fields: [...accumulator.fields.entries()].map(([name, type]) => ({ name, type })),
968
+ };
969
+ }
970
+ function tallyCategoryValue(tally, column, value) {
971
+ let columnTally = tally.get(column);
972
+ if (columnTally === undefined) {
973
+ columnTally = { counts: new Map(), overflowed: false };
974
+ tally.set(column, columnTally);
975
+ }
976
+ const key = value.length > MAX_CATEGORY_VALUE_LENGTH ? `${value.slice(0, MAX_CATEGORY_VALUE_LENGTH)}…` : value;
977
+ const existing = columnTally.counts.get(key);
978
+ if (existing !== undefined) {
979
+ columnTally.counts.set(key, existing + 1);
980
+ return;
981
+ }
982
+ if (columnTally.counts.size >= MAX_CATEGORY_DISTINCT) {
983
+ columnTally.overflowed = true;
984
+ return;
985
+ }
986
+ columnTally.counts.set(key, 1);
987
+ }
988
+ /**
989
+ * 文档型 Dataset 的 profile:没有行,所以不给行列统计,只给结构摘要。
990
+ *
991
+ * `$` 表示文档根,其余路径沿用 structure 的约定(对象字段用 `.name`、数组元素用 `[]`),
992
+ * 例如财务指标会得到 `abilities`(共 5 项)与 `abilities[].indicators`(共 40 项)。
993
+ */
994
+ function buildDocumentProfile(value) {
995
+ const entries = new Map();
996
+ collectStructure(value, '$', 0, entries);
997
+ const structure = {};
998
+ for (const [path, accumulator] of entries)
999
+ structure[path] = finishStructure(accumulator);
1000
+ const warnings = ['dataset is a document (no row array); profile describes structure only'];
1001
+ if (entries.size >= MAX_STRUCTURE_ENTRIES)
1002
+ warnings.push(`structure digest truncated at ${MAX_STRUCTURE_ENTRIES} paths`);
1003
+ if ([...entries.values()].some((accumulator) => accumulator.sampled))
1004
+ warnings.push(`structure digest sampled after ${MAX_STRUCTURE_SAMPLED_ELEMENTS} elements; sampled_elements is incomplete`);
1005
+ return {
1006
+ row_count: 0,
1007
+ columns: [],
1008
+ quality: { missing_values: 0, duplicate_rows: 0, time_ordered: null },
1009
+ structure,
1010
+ warnings,
1011
+ };
1012
+ }
1013
+ /**
1014
+ * 把文档内容裁到预算内:长数组按档位逐级截断(保留前 N 项),每次尝试都重新
1015
+ * 量体积,直到放得下。返回实际采用的档位与被省略的路径,供模型如实披露
1016
+ * 「完整共 N 项,本次只给了前 K 项」。
1017
+ */
1018
+ function boundDocument(value) {
1019
+ const measure = (candidate) => {
1020
+ try {
1021
+ const text = JSON.stringify(candidate);
1022
+ return typeof text === 'string' ? [...text].length : undefined;
1023
+ }
1024
+ catch {
1025
+ return undefined;
1026
+ }
1027
+ };
1028
+ const full = measure(value);
1029
+ if (full !== undefined && full <= MAX_DOCUMENT_CHARS)
1030
+ return { content: value, truncated: false, omitted: [] };
1031
+ for (const keep of DOCUMENT_ARRAY_KEEP_STEPS) {
1032
+ const omitted = [];
1033
+ const bounded = shrinkArrays(value, keep, '$', omitted, 0);
1034
+ const size = measure(bounded);
1035
+ if (size !== undefined && size <= MAX_DOCUMENT_CHARS)
1036
+ return { content: bounded, truncated: omitted.length > 0, omitted };
1037
+ }
1038
+ return {
1039
+ content: null,
1040
+ truncated: true,
1041
+ omitted: [{ path: '$', kept: 0, omitted: 1 }],
1042
+ };
1043
+ }
1044
+ function shrinkArrays(value, keep, path, omitted, depth) {
1045
+ if (depth > DOCUMENT_MAX_DEPTH)
1046
+ return value;
1047
+ if (Array.isArray(value)) {
1048
+ const head = value.slice(0, keep);
1049
+ if (value.length > keep && omitted.length < DOCUMENT_MAX_OMISSIONS) {
1050
+ omitted.push({ path, kept: keep, omitted: value.length - keep });
1051
+ }
1052
+ return head.map((element) => shrinkArrays(element, keep, `${path}[]`, omitted, depth + 1));
1053
+ }
1054
+ if (!isRecord(value))
1055
+ return value;
1056
+ const result = {};
1057
+ for (const [key, child] of Object.entries(value)) {
1058
+ result[key] = Array.isArray(child) || isRecord(child) ? shrinkArrays(child, keep, `${path}.${key}`, omitted, depth + 1) : child;
1059
+ }
1060
+ return result;
1061
+ }
1062
+ function profileCell(row, column) {
1063
+ if (isRecord(row))
1064
+ return { present: Object.prototype.hasOwnProperty.call(row, column), value: row[column] };
1065
+ return column === 'value' ? { present: true, value: row } : { present: false, value: undefined };
1066
+ }
1067
+ function profileObservedType(value) {
1068
+ if (value === null)
1069
+ return 'null';
1070
+ if (typeof value === 'boolean')
1071
+ return 'boolean';
1072
+ if (typeof value === 'number')
1073
+ return Number.isInteger(value) ? 'integer' : 'number';
1074
+ if (typeof value === 'string')
1075
+ return 'string';
1076
+ if (Array.isArray(value))
1077
+ return 'array';
1078
+ return 'object';
1079
+ }
1080
+ function incrementTypeCount(counts, type) {
1081
+ counts[type] = (counts[type] ?? 0) + 1;
1082
+ }
1083
+ function inferProfileType(observedTypes) {
1084
+ const nonNullTypes = Object.keys(observedTypes).filter((type) => !['missing', 'null'].includes(type) && (observedTypes[type] ?? 0) > 0);
1085
+ if (nonNullTypes.length === 1)
1086
+ return nonNullTypes[0];
1087
+ if (nonNullTypes.length > 1)
1088
+ return 'mixed';
1089
+ if ((observedTypes.null ?? 0) > 0)
1090
+ return 'null';
1091
+ return 'missing';
1092
+ }
1093
+ function isNumericContract(type) {
1094
+ return type === 'number' || type === 'integer';
1095
+ }
1096
+ function isFiniteNumericString(value) {
1097
+ return typeof value === 'string' && value.trim().length > 0 && Number.isFinite(Number(value));
1098
+ }
1099
+ function matchesProfileContract(value, contract) {
1100
+ if (value === null)
1101
+ return true;
1102
+ switch (contract.type) {
1103
+ case 'string': return typeof value === 'string';
1104
+ case 'number': return (typeof value === 'number' && Number.isFinite(value)) || (contract.allowNumericString === true && isFiniteNumericString(value));
1105
+ case 'integer': return (typeof value === 'number' && Number.isSafeInteger(value)) || (contract.allowNumericString === true && isFiniteNumericString(value) && Number.isSafeInteger(Number(value)));
1106
+ case 'boolean': return typeof value === 'boolean';
1107
+ case 'object': return isRecord(value);
1108
+ case 'array': return Array.isArray(value);
1109
+ case 'json': return true;
1110
+ }
1111
+ }
1112
+ function extractRowContracts(schema) {
1113
+ if (!isRecord(schema))
1114
+ return {};
1115
+ let rowSchema = schema;
1116
+ const rootProperties = isRecord(schema.properties) ? schema.properties : undefined;
1117
+ const itemSchema = rootProperties && isRecord(rootProperties.item) ? rootProperties.item : undefined;
1118
+ if (itemSchema)
1119
+ rowSchema = itemSchema;
1120
+ if (isRecord(rowSchema) && rowSchema.type === 'array' && rowSchema.items !== undefined)
1121
+ rowSchema = rowSchema.items;
1122
+ const properties = isRecord(rowSchema) && isRecord(rowSchema.properties) ? rowSchema.properties : undefined;
1123
+ if (!properties)
1124
+ return {};
1125
+ const contracts = {};
1126
+ for (const [column, property] of Object.entries(properties)) {
1127
+ const type = profileContractType(property);
1128
+ if (!type)
1129
+ continue;
1130
+ contracts[column] = { type, allowNumericString: type === 'number' || type === 'integer' };
1131
+ }
1132
+ return contracts;
1133
+ }
1134
+ function profileContractType(schema) {
1135
+ if (!isRecord(schema))
1136
+ return undefined;
1137
+ if (typeof schema.type === 'string') {
1138
+ if (schema.type === 'string')
1139
+ return 'string';
1140
+ if (schema.type === 'number')
1141
+ return 'number';
1142
+ if (schema.type === 'integer')
1143
+ return 'integer';
1144
+ if (schema.type === 'boolean')
1145
+ return 'boolean';
1146
+ if (schema.type === 'object')
1147
+ return 'object';
1148
+ if (schema.type === 'array')
1149
+ return 'array';
1150
+ }
1151
+ if (Array.isArray(schema.oneOf)) {
1152
+ for (const option of schema.oneOf) {
1153
+ const type = profileContractType(option);
1154
+ if (type)
1155
+ return type;
1156
+ }
1157
+ }
1158
+ return 'json';
1159
+ }
1160
+ function stableProfileValue(value) {
1161
+ if (value === null || typeof value !== 'object')
1162
+ return JSON.stringify(value);
1163
+ if (Array.isArray(value))
1164
+ return `[${value.map(stableProfileValue).join(',')}]`;
1165
+ const record = value;
1166
+ return `{${Object.keys(record).sort().map((key) => `${JSON.stringify(key)}:${stableProfileValue(record[key])}`).join(',')}}`;
1167
+ }
1168
+ function compareProfileValues(left, right) {
1169
+ if (typeof left === 'number' && typeof right === 'number')
1170
+ return left - right;
1171
+ const leftText = String(left);
1172
+ const rightText = String(right);
1173
+ return leftText < rightText ? -1 : leftText > rightText ? 1 : 0;
1174
+ }
1175
+ function profileQuantile(sorted, probability) {
1176
+ if (sorted.length === 1)
1177
+ return sorted[0];
1178
+ const position = (sorted.length - 1) * probability;
1179
+ const lower = Math.floor(position);
1180
+ const upper = Math.ceil(position);
1181
+ if (lower === upper)
1182
+ return sorted[lower];
1183
+ return sorted[lower] + (sorted[upper] - sorted[lower]) * (position - lower);
1184
+ }
1185
+ function validateStoredDatasetManifest(value, datasetId) {
1186
+ if (!isRecord(value))
1187
+ return undefined;
1188
+ if (value.dataset_id !== datasetId || !DATASET_ID_PATTERN.test(datasetId))
1189
+ return undefined;
1190
+ if (!(value.task_id === null || typeof value.task_id === 'string'))
1191
+ return undefined;
1192
+ if (typeof value.session_id !== 'string' || value.session_id.length === 0)
1193
+ return undefined;
1194
+ if (value.artifact_ref !== canonicalDatasetRef(datasetId))
1195
+ return undefined;
1196
+ if (value.format !== 'json' && value.format !== 'json_rows')
1197
+ return undefined;
1198
+ if (typeof value.capability !== 'string' || value.capability.length === 0)
1199
+ return undefined;
1200
+ if (typeof value.source_label !== 'string' || value.source_label.length === 0)
1201
+ return undefined;
1202
+ if (!(value.schema === null || isRecord(value.schema)))
1203
+ return undefined;
1204
+ if (!(value.row_count === null || validNonNegativeInteger(value.row_count)))
1205
+ return undefined;
1206
+ if (!Number.isSafeInteger(value.captured_at) || !Number.isSafeInteger(value.retention_until))
1207
+ return undefined;
1208
+ if (typeof value.params_digest !== 'string' || value.params_digest.length === 0)
1209
+ return undefined;
1210
+ if (value.row_key !== undefined && (typeof value.row_key !== 'string' || value.row_key.length === 0 || value.row_key.length > 128))
1211
+ return undefined;
1212
+ const ref = {
1213
+ dataset_id: value.dataset_id,
1214
+ task_id: value.task_id,
1215
+ session_id: value.session_id,
1216
+ artifact_ref: value.artifact_ref,
1217
+ format: value.format,
1218
+ capability: value.capability,
1219
+ source_label: value.source_label,
1220
+ schema: value.schema,
1221
+ row_count: value.row_count,
1222
+ captured_at: value.captured_at,
1223
+ retention_until: value.retention_until,
1224
+ params_digest: value.params_digest,
1225
+ };
1226
+ return {
1227
+ ...ref,
1228
+ // Manifests written before scope support remain readable by their producer only.
1229
+ session_scope_id: typeof value.session_scope_id === 'string' && value.session_scope_id.length > 0
1230
+ ? value.session_scope_id
1231
+ : ref.session_id,
1232
+ ...(value.row_key === undefined ? {} : { row_key: value.row_key }),
1233
+ };
1234
+ }
1235
+ function validateColumns(value, errorCode = 'profile_invalid', maxColumns = 64) {
1236
+ if (!Array.isArray(value) || value.length > maxColumns)
1237
+ throw new DatasetStoreError(errorCode, `columns must contain at most ${maxColumns} names`);
1238
+ const columns = value.map((column) => {
1239
+ if (typeof column !== 'string' || column.length === 0 || column.length > 128)
1240
+ throw new DatasetStoreError(errorCode, 'column name is invalid');
1241
+ return column;
1242
+ });
1243
+ if (new Set(columns).size !== columns.length)
1244
+ throw new DatasetStoreError(errorCode, 'columns must be unique');
1245
+ return columns;
1246
+ }
1247
+ function validateProfile(value) {
1248
+ if (!isRecord(value))
1249
+ throw new DatasetStoreError('profile_invalid', 'profile must be an object');
1250
+ const allowed = new Set(['row_count', 'columns', 'quality', 'statistics', 'categories', 'time_facts', 'structure', 'schema', 'validation', 'warnings']);
1251
+ if (Object.keys(value).some((key) => !allowed.has(key)))
1252
+ throw new DatasetStoreError('profile_invalid', 'profile contains unsupported fields');
1253
+ if (!validNonNegativeInteger(value.row_count))
1254
+ throw new DatasetStoreError('profile_invalid', 'profile.row_count must be a non-negative integer');
1255
+ const columns = validateColumns(value.columns, 'profile_invalid', Number.MAX_SAFE_INTEGER);
1256
+ if (!isRecord(value.quality))
1257
+ throw new DatasetStoreError('profile_invalid', 'profile.quality is required');
1258
+ const qualityKeys = new Set(['missing_values', 'duplicate_rows', 'time_ordered']);
1259
+ if (Object.keys(value.quality).some((key) => !qualityKeys.has(key)))
1260
+ throw new DatasetStoreError('profile_invalid', 'profile.quality contains unsupported fields');
1261
+ if (!validNonNegativeInteger(value.quality.missing_values) || !validNonNegativeInteger(value.quality.duplicate_rows))
1262
+ throw new DatasetStoreError('profile_invalid', 'quality counts must be non-negative integers');
1263
+ if (!(value.quality.time_ordered === null || typeof value.quality.time_ordered === 'boolean'))
1264
+ throw new DatasetStoreError('profile_invalid', 'quality.time_ordered must be boolean or null');
1265
+ if (!Array.isArray(value.warnings) || value.warnings.length > 32 || value.warnings.some((warning) => typeof warning !== 'string' || warning.length > 512))
1266
+ throw new DatasetStoreError('profile_invalid', 'warnings must be a bounded string array');
1267
+ let statistics;
1268
+ if (value.statistics !== undefined) {
1269
+ if (!isRecord(value.statistics))
1270
+ throw new DatasetStoreError('profile_invalid', 'statistics must be an object');
1271
+ statistics = {};
1272
+ const metricKeys = new Set(['count', 'sum', 'min', 'max', 'mean', 'p25', 'p50', 'p75']);
1273
+ for (const [column, metric] of Object.entries(value.statistics)) {
1274
+ if (column.length === 0 || column.length > 128 || !isRecord(metric) || Object.keys(metric).some((key) => !metricKeys.has(key)))
1275
+ throw new DatasetStoreError('profile_invalid', 'statistics contains an invalid column or metric');
1276
+ if (metric.count !== undefined && !validNonNegativeInteger(metric.count))
1277
+ throw new DatasetStoreError('profile_invalid', `statistics.${column}.count must be a non-negative integer`);
1278
+ for (const [key, item] of Object.entries(metric)) {
1279
+ if (key === 'count')
1280
+ continue;
1281
+ if (!(item === null || (typeof item === 'number' && Number.isFinite(item))))
1282
+ throw new DatasetStoreError('profile_invalid', `statistics.${column}.${key} must be a finite number or null`);
1283
+ }
1284
+ statistics[column] = metric;
1285
+ }
1286
+ }
1287
+ let categories;
1288
+ if (value.categories !== undefined) {
1289
+ if (!isRecord(value.categories))
1290
+ throw new DatasetStoreError('profile_invalid', 'categories must be an object');
1291
+ categories = {};
1292
+ for (const [column, item] of Object.entries(value.categories)) {
1293
+ if (!columns.includes(column) || !isRecord(item))
1294
+ throw new DatasetStoreError('profile_invalid', 'categories contains an invalid column');
1295
+ if (Object.keys(item).some((key) => !['distinct_count', 'top_values', 'truncated'].includes(key)))
1296
+ throw new DatasetStoreError('profile_invalid', `categories.${column} contains unsupported fields`);
1297
+ if (!(item.distinct_count === null || validNonNegativeInteger(item.distinct_count)))
1298
+ throw new DatasetStoreError('profile_invalid', `categories.${column}.distinct_count must be a non-negative integer or null`);
1299
+ if (typeof item.truncated !== 'boolean')
1300
+ throw new DatasetStoreError('profile_invalid', `categories.${column}.truncated must be boolean`);
1301
+ if (!Array.isArray(item.top_values) || item.top_values.length > MAX_CATEGORY_TOP_VALUES)
1302
+ throw new DatasetStoreError('profile_invalid', `categories.${column}.top_values must be a bounded array`);
1303
+ for (const entry of item.top_values) {
1304
+ if (!isRecord(entry) || typeof entry.value !== 'string' || entry.value.length > MAX_CATEGORY_VALUE_LENGTH + 1 || !validNonNegativeInteger(entry.count)) {
1305
+ throw new DatasetStoreError('profile_invalid', `categories.${column}.top_values contains an invalid entry`);
1306
+ }
1307
+ }
1308
+ categories[column] = {
1309
+ distinct_count: item.distinct_count,
1310
+ top_values: item.top_values,
1311
+ truncated: item.truncated,
1312
+ };
1313
+ }
1314
+ }
1315
+ let timeFacts;
1316
+ if (value.time_facts !== undefined) {
1317
+ const item = value.time_facts;
1318
+ if (!isRecord(item) || typeof item.time_column !== 'string' || !columns.includes(item.time_column))
1319
+ throw new DatasetStoreError('profile_invalid', 'time_facts.time_column must be a profile column');
1320
+ if (Object.keys(item).some((key) => !['time_column', 'ordered_ascending', 'covered_from', 'covered_to', 'covered_from_iso', 'covered_to_iso', 'axis', 'windows', 'first', 'last'].includes(key)))
1321
+ throw new DatasetStoreError('profile_invalid', 'time_facts contains unsupported fields');
1322
+ if (!(item.ordered_ascending === null || typeof item.ordered_ascending === 'boolean'))
1323
+ throw new DatasetStoreError('profile_invalid', 'time_facts.ordered_ascending must be boolean or null');
1324
+ if (!isRecord(item.first) || !isRecord(item.last))
1325
+ throw new DatasetStoreError('profile_invalid', 'time_facts.first and time_facts.last must be objects');
1326
+ for (const [key, entry] of [...Object.entries(item.first), ...Object.entries(item.last)]) {
1327
+ if (!columns.includes(key))
1328
+ throw new DatasetStoreError('profile_invalid', `time_facts.${key} is not a profile column`);
1329
+ if (entry !== null && typeof entry === 'object')
1330
+ throw new DatasetStoreError('profile_invalid', `time_facts.${key} must be a scalar value`);
1331
+ }
1332
+ const axis = item.axis;
1333
+ if (!isRecord(axis) || axis.column !== item.time_column || !['epoch_ms', 'date_string', 'unknown'].includes(String(axis.value_format))) {
1334
+ throw new DatasetStoreError('profile_invalid', 'time_facts.axis must describe the time column');
1335
+ }
1336
+ if (Object.keys(axis).some((key) => !['column', 'value_format', 'time_zone', 'utc_offset', 'aligned_to'].includes(key))) {
1337
+ throw new DatasetStoreError('profile_invalid', 'time_facts.axis contains unsupported fields');
1338
+ }
1339
+ const windows = item.windows;
1340
+ if (windows !== undefined) {
1341
+ if (!Array.isArray(windows) || windows.length > MAX_TIME_WINDOWS)
1342
+ throw new DatasetStoreError('profile_invalid', `time_facts.windows must contain at most ${MAX_TIME_WINDOWS} windows`);
1343
+ for (const window of windows) {
1344
+ if (!isRecord(window) || typeof window.name !== 'string' || typeof window.start_date !== 'string' || typeof window.end_date !== 'string') {
1345
+ throw new DatasetStoreError('profile_invalid', 'time_facts.windows items must carry name/start_date/end_date');
1346
+ }
1347
+ }
1348
+ }
1349
+ timeFacts = {
1350
+ time_column: item.time_column,
1351
+ ordered_ascending: item.ordered_ascending,
1352
+ covered_from: item.covered_from ?? null,
1353
+ covered_to: item.covered_to ?? null,
1354
+ ...(typeof item.covered_from_iso === 'string' ? { covered_from_iso: item.covered_from_iso } : {}),
1355
+ ...(typeof item.covered_to_iso === 'string' ? { covered_to_iso: item.covered_to_iso } : {}),
1356
+ axis: {
1357
+ column: item.time_column,
1358
+ value_format: axis.value_format,
1359
+ ...(typeof axis.time_zone === 'string' ? { time_zone: axis.time_zone } : {}),
1360
+ ...(typeof axis.utc_offset === 'string' ? { utc_offset: axis.utc_offset } : {}),
1361
+ ...(axis.aligned_to === 'local_midnight' || axis.aligned_to === 'calendar_day' || axis.aligned_to === 'unknown'
1362
+ ? { aligned_to: axis.aligned_to }
1363
+ : {}),
1364
+ },
1365
+ ...(windows === undefined ? {} : { windows: windows }),
1366
+ first: item.first,
1367
+ last: item.last,
1368
+ };
1369
+ }
1370
+ let structure;
1371
+ if (value.structure !== undefined) {
1372
+ if (!isRecord(value.structure))
1373
+ throw new DatasetStoreError('profile_invalid', 'structure must be an object');
1374
+ structure = {};
1375
+ if (Object.keys(value.structure).length > MAX_STRUCTURE_ENTRIES)
1376
+ throw new DatasetStoreError('profile_invalid', `structure must contain at most ${MAX_STRUCTURE_ENTRIES} paths`);
1377
+ for (const [path, item] of Object.entries(value.structure)) {
1378
+ if (path.length === 0 || path.length > 256 || !isRecord(item))
1379
+ throw new DatasetStoreError('profile_invalid', 'structure contains an invalid path');
1380
+ if (Object.keys(item).some((key) => !['type', 'occurrences', 'total_elements', 'sampled_elements', 'empty_count', 'min_length', 'max_length', 'fields'].includes(key)))
1381
+ throw new DatasetStoreError('profile_invalid', `structure.${path} contains unsupported fields`);
1382
+ if (item.type !== 'array' && item.type !== 'object')
1383
+ throw new DatasetStoreError('profile_invalid', `structure.${path}.type is invalid`);
1384
+ const hasTotal = item.total_elements !== undefined;
1385
+ const hasSampled = item.sampled_elements !== undefined;
1386
+ if (hasTotal === hasSampled)
1387
+ throw new DatasetStoreError("profile_invalid", "structure path must contain exactly one of total_elements or sampled_elements");
1388
+ for (const key of ['occurrences', 'empty_count', 'min_length', 'max_length', ...(hasTotal ? ['total_elements'] : ['sampled_elements'])]) {
1389
+ if (!validNonNegativeInteger(item[key]))
1390
+ throw new DatasetStoreError("profile_invalid", "structure field must be a non-negative integer");
1391
+ }
1392
+ if (!Array.isArray(item.fields) || item.fields.length > MAX_STRUCTURE_FIELDS)
1393
+ throw new DatasetStoreError('profile_invalid', `structure.${path}.fields must be a bounded array`);
1394
+ for (const field of item.fields) {
1395
+ if (!isRecord(field) || typeof field.name !== 'string' || field.name.length === 0 || field.name.length > 128 || typeof field.type !== 'string' || field.type.length === 0 || field.type.length > 32) {
1396
+ throw new DatasetStoreError('profile_invalid', `structure.${path}.fields contains an invalid entry`);
1397
+ }
1398
+ }
1399
+ structure[path] = item;
1400
+ }
1401
+ }
1402
+ let schema;
1403
+ if (value.schema !== undefined) {
1404
+ if (!isRecord(value.schema))
1405
+ throw new DatasetStoreError('profile_invalid', 'schema must be an object');
1406
+ schema = {};
1407
+ const observedTypes = new Set(['missing', 'null', 'boolean', 'integer', 'number', 'string', 'object', 'array', 'mixed']);
1408
+ const contractTypes = new Set(['string', 'number', 'integer', 'boolean', 'object', 'array', 'json']);
1409
+ for (const [column, item] of Object.entries(value.schema)) {
1410
+ if (column.length === 0 || column.length > 128 || !isRecord(item))
1411
+ throw new DatasetStoreError('profile_invalid', 'schema contains an invalid column');
1412
+ const itemKeys = new Set(['contract_type', 'inferred_type', 'observed_types', 'missing_count', 'null_count', 'non_null_count', 'invalid_count', 'nullable', 'contract_nullable', 'required', 'allow_numeric_string', 'numeric_string_count']);
1413
+ if (Object.keys(item).some((key) => !itemKeys.has(key)))
1414
+ throw new DatasetStoreError('profile_invalid', `schema.${column} contains unsupported fields`);
1415
+ if (!(item.contract_type === null || (typeof item.contract_type === 'string' && contractTypes.has(item.contract_type))))
1416
+ throw new DatasetStoreError('profile_invalid', `schema.${column}.contract_type is invalid`);
1417
+ if (typeof item.inferred_type !== 'string' || !observedTypes.has(item.inferred_type))
1418
+ throw new DatasetStoreError('profile_invalid', `schema.${column}.inferred_type is invalid`);
1419
+ const observedTypeCounts = item.observed_types;
1420
+ if (!isRecord(observedTypeCounts) || Object.keys(observedTypeCounts).some((key) => !observedTypes.has(key) || !validNonNegativeInteger(observedTypeCounts[key])))
1421
+ throw new DatasetStoreError('profile_invalid', `schema.${column}.observed_types is invalid`);
1422
+ const observedTotal = Object.values(observedTypeCounts).reduce((sum, count) => sum + count, 0);
1423
+ if (observedTotal !== value.row_count)
1424
+ throw new DatasetStoreError('profile_invalid', `schema.${column}.observed_types count does not equal row_count`);
1425
+ for (const key of ['missing_count', 'null_count', 'non_null_count', 'invalid_count'])
1426
+ if (!validNonNegativeInteger(item[key]))
1427
+ throw new DatasetStoreError('profile_invalid', `schema.${column}.${key} must be a non-negative integer`);
1428
+ const rowCount = value.row_count;
1429
+ const missingCount = item.missing_count;
1430
+ const nullCount = item.null_count;
1431
+ const nonNullCount = item.non_null_count;
1432
+ const invalidCount = item.invalid_count;
1433
+ if (missingCount + nullCount + nonNullCount !== rowCount)
1434
+ throw new DatasetStoreError('profile_invalid', `schema.${column} value counts do not equal row_count`);
1435
+ if (invalidCount > nonNullCount)
1436
+ throw new DatasetStoreError('profile_invalid', `schema.${column}.invalid_count exceeds non_null_count`);
1437
+ if (typeof item.nullable !== 'boolean')
1438
+ throw new DatasetStoreError('profile_invalid', `schema.${column}.nullable must be boolean`);
1439
+ if (item.nullable !== (nullCount > 0))
1440
+ throw new DatasetStoreError('profile_invalid', `schema.${column}.nullable does not match null_count`);
1441
+ if (item.contract_nullable !== undefined && typeof item.contract_nullable !== 'boolean')
1442
+ throw new DatasetStoreError('profile_invalid', `schema.${column}.contract_nullable must be boolean`);
1443
+ if (item.required !== undefined && typeof item.required !== 'boolean')
1444
+ throw new DatasetStoreError('profile_invalid', `schema.${column}.required must be boolean`);
1445
+ if (item.allow_numeric_string !== undefined && typeof item.allow_numeric_string !== 'boolean')
1446
+ throw new DatasetStoreError('profile_invalid', `schema.${column}.allow_numeric_string must be boolean`);
1447
+ if (item.numeric_string_count !== undefined && !validNonNegativeInteger(item.numeric_string_count))
1448
+ throw new DatasetStoreError('profile_invalid', `schema.${column}.numeric_string_count must be a non-negative integer`);
1449
+ if (item.numeric_string_count !== undefined && item.numeric_string_count > nonNullCount)
1450
+ throw new DatasetStoreError('profile_invalid', `schema.${column}.numeric_string_count exceeds non_null_count`);
1451
+ schema[column] = item;
1452
+ }
1453
+ if (Object.keys(schema).some((column) => !columns.includes(column)))
1454
+ throw new DatasetStoreError('profile_invalid', 'schema columns must be listed in profile.columns');
1455
+ }
1456
+ let validation;
1457
+ if (value.validation !== undefined) {
1458
+ if (!isRecord(value.validation) || (value.validation.status !== 'pass' && value.validation.status !== 'fail') || !Array.isArray(value.validation.violations) || value.validation.violations.length > 64) {
1459
+ throw new DatasetStoreError('profile_invalid', 'validation must contain a status and bounded violations');
1460
+ }
1461
+ const violations = [];
1462
+ for (const item of value.validation.violations) {
1463
+ if (!isRecord(item) || typeof item.column !== 'string' || item.column.length === 0 || !['required', 'nullable', 'type'].includes(String(item.rule)) || !(typeof item.expected === 'string' || typeof item.expected === 'boolean') || typeof item.observed !== 'string' || !validNonNegativeInteger(item.invalid_count) || !['error', 'warning'].includes(String(item.severity))) {
1464
+ throw new DatasetStoreError('profile_invalid', 'validation contains an invalid violation');
1465
+ }
1466
+ violations.push({
1467
+ column: item.column,
1468
+ rule: item.rule,
1469
+ expected: item.expected,
1470
+ observed: item.observed,
1471
+ invalid_count: item.invalid_count,
1472
+ severity: item.severity,
1473
+ });
1474
+ }
1475
+ if ((value.validation.status === 'pass' && violations.length > 0) || (value.validation.status === 'fail' && violations.length === 0)) {
1476
+ throw new DatasetStoreError('profile_invalid', 'validation status does not match violations');
1477
+ }
1478
+ validation = { status: value.validation.status, violations };
1479
+ }
1480
+ return { row_count: value.row_count, columns, quality: { missing_values: value.quality.missing_values, duplicate_rows: value.quality.duplicate_rows, time_ordered: value.quality.time_ordered }, ...(statistics ? { statistics } : {}), ...(categories ? { categories } : {}), ...(timeFacts ? { time_facts: timeFacts } : {}), ...(structure ? { structure } : {}), ...(schema ? { schema } : {}), ...(validation ? { validation } : {}), warnings: value.warnings };
1481
+ }
1482
+ export function provideWorkspaceDatasetStore(ctx, options) {
1483
+ const store = new WorkspaceDatasetStore(options);
1484
+ ctx.provide('datasetStore', store);
1485
+ return store;
1486
+ }