@kaelio/ktx 0.13.1 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/python/{kaelio_ktx-0.13.1-py3-none-any.whl → kaelio_ktx-0.15.0-py3-none-any.whl} +0 -0
- package/assets/python/manifest.json +4 -4
- package/dist/.tsbuildinfo +1 -1
- package/dist/cli-program.js +1 -1
- package/dist/commands/ingest-commands.d.ts +9 -0
- package/dist/commands/ingest-commands.js +37 -1
- package/dist/commands/knowledge-commands.js +3 -0
- package/dist/commands/setup-commands.js +15 -1
- package/dist/connection-drivers.d.ts +2 -0
- package/dist/connection-drivers.js +7 -3
- package/dist/connection.d.ts +3 -0
- package/dist/connection.js +27 -0
- package/dist/connectors/bigquery/connector.d.ts +17 -6
- package/dist/connectors/bigquery/connector.js +152 -76
- package/dist/connectors/bigquery/dialect.d.ts +2 -2
- package/dist/connectors/clickhouse/connector.d.ts +1 -0
- package/dist/connectors/clickhouse/connector.js +45 -16
- package/dist/connectors/clickhouse/dialect.d.ts +2 -2
- package/dist/connectors/mongodb/connector.d.ts +69 -0
- package/dist/connectors/mongodb/connector.js +307 -0
- package/dist/connectors/mongodb/dialect.d.ts +17 -0
- package/dist/connectors/mongodb/dialect.js +50 -0
- package/dist/connectors/mongodb/live-database-introspection.d.ts +10 -0
- package/dist/connectors/mongodb/live-database-introspection.js +24 -0
- package/dist/connectors/mongodb/schema-inference.d.ts +20 -0
- package/dist/connectors/mongodb/schema-inference.js +110 -0
- package/dist/connectors/mysql/connector.d.ts +1 -0
- package/dist/connectors/mysql/connector.js +18 -2
- package/dist/connectors/mysql/dialect.d.ts +2 -2
- package/dist/connectors/postgres/connector.d.ts +1 -0
- package/dist/connectors/postgres/connector.js +20 -3
- package/dist/connectors/postgres/dialect.d.ts +2 -2
- package/dist/connectors/snowflake/connector.d.ts +1 -0
- package/dist/connectors/snowflake/connector.js +27 -3
- package/dist/connectors/snowflake/dialect.d.ts +2 -2
- package/dist/connectors/sqlite/connector.d.ts +11 -1
- package/dist/connectors/sqlite/connector.js +120 -17
- package/dist/connectors/sqlite/dialect.d.ts +2 -2
- package/dist/connectors/sqlite/read-query-child.d.ts +1 -0
- package/dist/connectors/sqlite/read-query-child.js +23 -0
- package/dist/connectors/sqlserver/connector.d.ts +2 -0
- package/dist/connectors/sqlserver/connector.js +20 -3
- package/dist/connectors/sqlserver/dialect.d.ts +2 -2
- package/dist/context/cache/content-result-cache.d.ts +36 -0
- package/dist/context/cache/content-result-cache.js +14 -0
- package/dist/context/cache/sqlite-content-result-cache.d.ts +18 -0
- package/dist/context/cache/sqlite-content-result-cache.js +223 -0
- package/dist/context/connections/bigquery-identifiers.d.ts +1 -0
- package/dist/context/connections/bigquery-identifiers.js +7 -0
- package/dist/context/connections/configured-connections.d.ts +9 -0
- package/dist/context/connections/configured-connections.js +18 -0
- package/dist/context/connections/dialects.d.ts +30 -4
- package/dist/context/connections/dialects.js +38 -11
- package/dist/context/connections/drivers.js +21 -0
- package/dist/context/connections/gdrive-config.d.ts +19 -0
- package/dist/context/connections/gdrive-config.js +57 -0
- package/dist/context/connections/query-deadline.d.ts +31 -0
- package/dist/context/connections/query-deadline.js +37 -0
- package/dist/context/ingest/adapters/gdrive/chunk.d.ts +3 -0
- package/dist/context/ingest/adapters/gdrive/chunk.js +74 -0
- package/dist/context/ingest/adapters/gdrive/detect.d.ts +1 -0
- package/dist/context/ingest/adapters/gdrive/detect.js +20 -0
- package/dist/context/ingest/adapters/gdrive/fetch.d.ts +6 -0
- package/dist/context/ingest/adapters/gdrive/fetch.js +95 -0
- package/dist/context/ingest/adapters/gdrive/gdrive-client.d.ts +30 -0
- package/dist/context/ingest/adapters/gdrive/gdrive-client.js +132 -0
- package/dist/context/ingest/adapters/gdrive/gdrive.adapter.d.ts +11 -0
- package/dist/context/ingest/adapters/gdrive/gdrive.adapter.js +27 -0
- package/dist/context/ingest/adapters/gdrive/normalize.d.ts +2 -0
- package/dist/context/ingest/adapters/gdrive/normalize.js +256 -0
- package/dist/context/ingest/adapters/gdrive/types.d.ts +153 -0
- package/dist/context/ingest/adapters/gdrive/types.js +36 -0
- package/dist/context/ingest/adapters/live-database/daemon-introspection.js +24 -0
- package/dist/context/ingest/adapters/live-database/fetch-report.d.ts +8 -0
- package/dist/context/ingest/adapters/live-database/fetch-report.js +37 -0
- package/dist/context/ingest/adapters/live-database/live-database.adapter.d.ts +2 -1
- package/dist/context/ingest/adapters/live-database/live-database.adapter.js +9 -2
- package/dist/context/ingest/adapters/live-database/manifest.d.ts +2 -0
- package/dist/context/ingest/adapters/live-database/manifest.js +8 -4
- package/dist/context/ingest/adapters/live-database/scan-outcome.d.ts +17 -0
- package/dist/context/ingest/adapters/live-database/scan-outcome.js +40 -0
- package/dist/context/ingest/adapters/live-database/stage.js +5 -5
- package/dist/context/ingest/adapters/metabase/types.d.ts +1 -1
- package/dist/context/ingest/adapters/metabase/types.js +1 -1
- package/dist/context/ingest/adapters/sigma/chunk.d.ts +6 -0
- package/dist/context/ingest/adapters/sigma/chunk.js +119 -0
- package/dist/context/ingest/adapters/sigma/client-port.d.ts +45 -0
- package/dist/context/ingest/adapters/sigma/client-port.js +1 -0
- package/dist/context/ingest/adapters/sigma/client.d.ts +33 -0
- package/dist/context/ingest/adapters/sigma/client.js +176 -0
- package/dist/context/ingest/adapters/sigma/detect.d.ts +1 -0
- package/dist/context/ingest/adapters/sigma/detect.js +23 -0
- package/dist/context/ingest/adapters/sigma/fetch.d.ts +14 -0
- package/dist/context/ingest/adapters/sigma/fetch.js +191 -0
- package/dist/context/ingest/adapters/sigma/local-sigma.adapter.d.ts +17 -0
- package/dist/context/ingest/adapters/sigma/local-sigma.adapter.js +41 -0
- package/dist/context/ingest/adapters/sigma/project.d.ts +8 -0
- package/dist/context/ingest/adapters/sigma/project.js +186 -0
- package/dist/context/ingest/adapters/sigma/sigma.adapter.d.ts +18 -0
- package/dist/context/ingest/adapters/sigma/sigma.adapter.js +43 -0
- package/dist/context/ingest/adapters/sigma/types.d.ts +80 -0
- package/dist/context/ingest/adapters/sigma/types.js +82 -0
- package/dist/context/ingest/artifact-gates.d.ts +32 -1
- package/dist/context/ingest/artifact-gates.js +85 -18
- package/dist/context/ingest/final-gate-prune.d.ts +35 -0
- package/dist/context/ingest/final-gate-prune.js +229 -0
- package/dist/context/ingest/ingest-bundle.runner.d.ts +3 -0
- package/dist/context/ingest/ingest-bundle.runner.js +459 -240
- package/dist/context/ingest/isolated-diff/patch-integrator.d.ts +0 -11
- package/dist/context/ingest/isolated-diff/patch-integrator.js +0 -44
- package/dist/context/ingest/isolated-diff/work-unit-executor.d.ts +1 -0
- package/dist/context/ingest/isolated-diff/work-unit-executor.js +13 -4
- package/dist/context/ingest/local-adapters.d.ts +2 -1
- package/dist/context/ingest/local-adapters.js +28 -0
- package/dist/context/ingest/local-bundle-runtime.js +7 -2
- package/dist/context/ingest/local-ingest.js +1 -1
- package/dist/context/ingest/local-stage-ingest.d.ts +3 -1
- package/dist/context/ingest/local-stage-ingest.js +3 -1
- package/dist/context/ingest/ports.d.ts +3 -0
- package/dist/context/ingest/report-snapshot.js +13 -3
- package/dist/context/ingest/reports.d.ts +3 -3
- package/dist/context/ingest/reports.js +3 -1
- package/dist/context/ingest/stages/build-wu-context.d.ts +1 -0
- package/dist/context/ingest/stages/build-wu-context.js +2 -1
- package/dist/context/ingest/stages/stage-3-work-units.d.ts +2 -1
- package/dist/context/ingest/stages/stage-3-work-units.js +4 -10
- package/dist/context/ingest/stages/validate-wu-sources.d.ts +12 -0
- package/dist/context/ingest/stages/validate-wu-sources.js +23 -8
- package/dist/context/ingest/tools/read-raw-file.tool.js +10 -5
- package/dist/context/ingest/tools/read-raw-span.tool.js +10 -5
- package/dist/context/ingest/tools/warehouse-verification/discover-data.tool.js +1 -1
- package/dist/context/ingest/tools/warehouse-verification/entity-details.tool.js +1 -1
- package/dist/context/ingest/tools/warehouse-verification/sql-execution.tool.js +1 -1
- package/dist/context/ingest/types.d.ts +3 -0
- package/dist/context/ingest/wiki-body-refs.d.ts +28 -0
- package/dist/context/ingest/wiki-body-refs.js +37 -8
- package/dist/context/ingest/wiki-sl-ref-repair.d.ts +1 -0
- package/dist/context/ingest/wiki-sl-ref-repair.js +4 -0
- package/dist/context/ingest/work-unit-cache.d.ts +67 -0
- package/dist/context/ingest/work-unit-cache.js +230 -0
- package/dist/context/llm/ai-sdk-runtime.d.ts +1 -0
- package/dist/context/llm/ai-sdk-runtime.js +5 -0
- package/dist/context/llm/claude-code-runtime.d.ts +4 -1
- package/dist/context/llm/claude-code-runtime.js +38 -8
- package/dist/context/llm/codex-runtime.d.ts +4 -1
- package/dist/context/llm/codex-runtime.js +48 -35
- package/dist/context/llm/runtime-port.d.ts +26 -0
- package/dist/context/llm/subprocess-generate-object-child.d.ts +1 -0
- package/dist/context/llm/subprocess-generate-object-child.js +34 -0
- package/dist/context/llm/subprocess-generate-object.d.ts +42 -0
- package/dist/context/llm/subprocess-generate-object.js +122 -0
- package/dist/context/mcp/context-tools.d.ts +2 -0
- package/dist/context/mcp/context-tools.js +111 -17
- package/dist/context/mcp/local-project-ports.d.ts +7 -0
- package/dist/context/mcp/local-project-ports.js +29 -0
- package/dist/context/mcp/logger.d.ts +24 -0
- package/dist/context/mcp/logger.js +49 -0
- package/dist/context/mcp/server.js +2 -0
- package/dist/context/mcp/types.d.ts +17 -0
- package/dist/context/memory/memory-agent.service.js +1 -1
- package/dist/context/project/config.d.ts +72 -0
- package/dist/context/project/config.js +5 -0
- package/dist/context/project/driver-schemas.d.ts +35 -0
- package/dist/context/project/driver-schemas.js +89 -1
- package/dist/context/project/project.js +1 -1
- package/dist/context/project/setup-config.js +1 -0
- package/dist/context/scan/description-generation.d.ts +4 -0
- package/dist/context/scan/description-generation.js +100 -13
- package/dist/context/scan/enabled-tables.d.ts +5 -0
- package/dist/context/scan/enabled-tables.js +13 -1
- package/dist/context/scan/enrichment-state.d.ts +50 -7
- package/dist/context/scan/enrichment-state.js +30 -13
- package/dist/context/scan/local-enrichment-artifacts.d.ts +41 -0
- package/dist/context/scan/local-enrichment-artifacts.js +155 -32
- package/dist/context/scan/local-enrichment.d.ts +39 -4
- package/dist/context/scan/local-enrichment.js +345 -92
- package/dist/context/scan/local-scan.d.ts +4 -1
- package/dist/context/scan/local-scan.js +54 -13
- package/dist/context/scan/local-structural-artifacts.d.ts +3 -1
- package/dist/context/scan/local-structural-artifacts.js +5 -0
- package/dist/context/scan/object-introspection.d.ts +21 -0
- package/dist/context/scan/object-introspection.js +35 -0
- package/dist/context/scan/relationship-benchmarks.js +5 -3
- package/dist/context/scan/relationship-composite-candidates.d.ts +6 -3
- package/dist/context/scan/relationship-composite-candidates.js +13 -1
- package/dist/context/scan/relationship-detection-budget.d.ts +35 -0
- package/dist/context/scan/relationship-detection-budget.js +53 -0
- package/dist/context/scan/relationship-diagnostics.d.ts +5 -0
- package/dist/context/scan/relationship-diagnostics.js +2 -0
- package/dist/context/scan/relationship-discovery.d.ts +9 -3
- package/dist/context/scan/relationship-discovery.js +27 -4
- package/dist/context/scan/relationship-llm-proposal.js +36 -27
- package/dist/context/scan/relationship-profiling.d.ts +7 -3
- package/dist/context/scan/relationship-profiling.js +53 -41
- package/dist/context/scan/relationship-validation.d.ts +6 -4
- package/dist/context/scan/relationship-validation.js +46 -32
- package/dist/context/scan/sqlite-local-enrichment-state-store.d.ts +8 -1
- package/dist/context/scan/sqlite-local-enrichment-state-store.js +71 -159
- package/dist/context/scan/types.d.ts +5 -3
- package/dist/context/scan/types.js +2 -0
- package/dist/context/sl/local-query.js +5 -0
- package/dist/context/sl/pglite-sl-search-prototype.js +1 -1
- package/dist/context/sl/semantic-layer.service.js +1 -1
- package/dist/context/sl/source-files.d.ts +5 -0
- package/dist/context/sl/source-files.js +17 -11
- package/dist/context/sl/tools/connection-id-schema.js +1 -1
- package/dist/context/sql-analysis/dialect-notes.d.ts +9 -0
- package/dist/context/sql-analysis/dialect-notes.js +40 -0
- package/dist/context/sql-analysis/dialects/bigquery.md +13 -0
- package/dist/context/sql-analysis/dialects/clickhouse.md +9 -0
- package/dist/context/sql-analysis/dialects/mysql.md +9 -0
- package/dist/context/sql-analysis/dialects/postgres.md +10 -0
- package/dist/context/sql-analysis/dialects/snowflake.md +10 -0
- package/dist/context/sql-analysis/dialects/sqlite.md +11 -0
- package/dist/context/sql-analysis/dialects/tsql.md +10 -0
- package/dist/context/wiki/keys.js +1 -1
- package/dist/context/wiki/knowledge-wiki.service.d.ts +4 -4
- package/dist/context/wiki/knowledge-wiki.service.js +2 -1
- package/dist/context/wiki/local-knowledge.d.ts +13 -0
- package/dist/context/wiki/local-knowledge.js +50 -6
- package/dist/context/wiki/sqlite-knowledge-index.d.ts +2 -0
- package/dist/context/wiki/sqlite-knowledge-index.js +21 -5
- package/dist/context/wiki/tools/wiki-write.tool.d.ts +2 -0
- package/dist/context/wiki/tools/wiki-write.tool.js +27 -0
- package/dist/context/wiki/types.d.ts +6 -0
- package/dist/context-build-view.d.ts +3 -0
- package/dist/context-build-view.js +1 -0
- package/dist/knowledge.d.ts +2 -0
- package/dist/knowledge.js +12 -1
- package/dist/local-adapters.js +11 -0
- package/dist/mcp-http-server.js +15 -1
- package/dist/mcp-server-factory.d.ts +2 -0
- package/dist/mcp-server-factory.js +15 -1
- package/dist/mcp-stdio-server.js +10 -2
- package/dist/notion-page-picker.js +1 -1
- package/dist/prompts/memory_agent_external_ingest.md +2 -0
- package/dist/public-ingest.d.ts +3 -1
- package/dist/public-ingest.js +4 -0
- package/dist/scan.d.ts +3 -1
- package/dist/scan.js +7 -0
- package/dist/setup-databases.d.ts +1 -1
- package/dist/setup-databases.js +43 -1
- package/dist/setup-sources.d.ts +6 -1
- package/dist/setup-sources.js +208 -48
- package/dist/setup.d.ts +3 -0
- package/dist/setup.js +6 -1
- package/dist/skills/analytics/SKILL.md +200 -2
- package/dist/skills/gdrive_synthesize/SKILL.md +97 -0
- package/dist/skills/sigma_ingest/SKILL.md +189 -0
- package/dist/skills/wiki_capture/SKILL.md +24 -0
- package/dist/sql.js +4 -0
- package/dist/status-project.d.ts +4 -0
- package/dist/status-project.js +54 -2
- package/dist/telemetry/events.d.ts +2 -2
- package/dist/text-ingest.d.ts +4 -0
- package/dist/text-ingest.js +58 -29
- package/dist/verbatim-ingest.d.ts +46 -0
- package/dist/verbatim-ingest.js +193 -0
- package/package.json +5 -1
- package/dist/context/ingest/final-gate-repair.d.ts +0 -28
- package/dist/context/ingest/final-gate-repair.js +0 -91
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: gdrive_synthesize
|
|
3
|
+
description: Synthesize durable KTX wiki pages from staged Google Drive document pulls. Load when a WorkUnit contains Google Doc raw files from `docs/**`.
|
|
4
|
+
callers: [memory_agent]
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
# Google Drive Doc Synthesis
|
|
8
|
+
|
|
9
|
+
Use this skill when a WorkUnit contains staged Google Drive content from `docs/**`.
|
|
10
|
+
|
|
11
|
+
## Role
|
|
12
|
+
|
|
13
|
+
Each WorkUnit is one Google Doc plus its metadata. Read the assigned raw files, then write a small set of durable wiki entries that capture reusable organizational knowledge. Write final memory directly; do not write candidates.
|
|
14
|
+
|
|
15
|
+
## Required Workflow
|
|
16
|
+
|
|
17
|
+
1. Read the WorkUnit notes and `rawFiles` list. Document content lives in `page.md`; `metadata.json` holds title, path, url, modified time, and Drive folder context.
|
|
18
|
+
2. For each assigned doc, call `read_raw_file`, or `read_raw_span` for oversized docs when the notes specify a span.
|
|
19
|
+
3. Search `wiki_search` for existing pages that overlap the WorkUnit topics. Prefer updating an existing page over creating a duplicate.
|
|
20
|
+
4. Use `context_evidence_search`, `context_evidence_read`, and `context_evidence_neighbors` when indexed document chunks would help reconcile related facts. Pass `chunkId` and `documentId` values verbatim as returned by the evidence tools.
|
|
21
|
+
5. Write durable business knowledge with `wiki_write`. Aim for a small number of high-quality pages per doc. Include `rawPaths` with the exact Google Drive raw files that support each page.
|
|
22
|
+
6. If a doc references warehouse, dbt, Looker, Metabase, or MetricFlow objects, you may verify them with `discover_data`, `entity_details`, `sql_execution`, `sl_discover`, or `sl_read_source`, but Google Drive docs are knowledge-only in v1. Do not create semantic-layer sources under the `gdrive` connection.
|
|
23
|
+
7. For every deleted raw path in the Eviction Set, call `eviction_list`, decide retention, then `emit_eviction_decision`. Do this even when no wiki write is needed.
|
|
24
|
+
|
|
25
|
+
## What To Capture
|
|
26
|
+
|
|
27
|
+
Capture durable, reusable company knowledge:
|
|
28
|
+
|
|
29
|
+
- policies, workflows, process rules, ownership conventions, and operating procedures
|
|
30
|
+
- product definitions, business terminology, and organizational guidance
|
|
31
|
+
- source-of-truth statements, caveats, conflict notes, and supersession guidance
|
|
32
|
+
- cross-system aliases that connect doc terminology to warehouse, dbt, Looker, Metabase, or MetricFlow names
|
|
33
|
+
|
|
34
|
+
Skip noisy or transient content:
|
|
35
|
+
|
|
36
|
+
- brainstorming notes with no durable rule
|
|
37
|
+
- task lists, meeting scheduling details, and time-bounded status updates
|
|
38
|
+
- duplicate docs with no new fact
|
|
39
|
+
- shallow summaries that add no reusable policy or definition
|
|
40
|
+
|
|
41
|
+
## Quality
|
|
42
|
+
|
|
43
|
+
Prefer fewer, stronger entries. Every wiki entry must cite at least one Google Doc using its title or path and last modified date when available. When evidence conflicts, write a conflict note inside the wiki page rather than choosing silently.
|
|
44
|
+
|
|
45
|
+
If one doc covers several related ideas, synthesize the shared durable rules instead of writing one thin page per paragraph. For oversized spans, read only the assigned span unless the WorkUnit explicitly asks for neighboring context.
|
|
46
|
+
|
|
47
|
+
Search existing wiki pages for the same `tables:` or `sl_refs:` frontmatter and for source-of-truth aliases before creating a new page. If an existing page already documents the same warehouse object or business concept, update it instead of creating a differently named duplicate.
|
|
48
|
+
|
|
49
|
+
## Citation Style
|
|
50
|
+
|
|
51
|
+
```md
|
|
52
|
+
## Agentic Harness
|
|
53
|
+
- The harness provides the operational framework that turns an agent prototype into a production system.
|
|
54
|
+
- Source: Google Doc - Herness, last modified 2026-05-24.
|
|
55
|
+
- Conflict note: An older internal note uses a narrower definition focused only on tool wiring; treat the current Google Doc as the durable operating definition unless replaced explicitly.
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
## Semantic-Layer Rules
|
|
59
|
+
|
|
60
|
+
- Google Drive docs are knowledge-only in v1; keep durable output in wiki pages.
|
|
61
|
+
- Do not create semantic-layer sources under the `gdrive` connection.
|
|
62
|
+
- If a doc references an existing warehouse or semantic-layer object and you can verify it, you may attach `sl_refs` in wiki output after confirmation.
|
|
63
|
+
- If a doc mentions a table or source that cannot be verified, keep the identifier in wiki text as unverified or use `emit_unmapped_fallback` only when the missing physical object itself is the important durable fact.
|
|
64
|
+
|
|
65
|
+
## Identifier Verification Protocol
|
|
66
|
+
|
|
67
|
+
Before writing a wiki page on any topic:
|
|
68
|
+
|
|
69
|
+
1. `discover_data({query: "<topic>"})` - see what wikis, SL sources, and raw
|
|
70
|
+
tables already exist. Prefer updating existing pages over creating new ones.
|
|
71
|
+
|
|
72
|
+
Before emitting any `schema.table` or `schema.table.column` into a wiki body,
|
|
73
|
+
`tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`:
|
|
74
|
+
|
|
75
|
+
2. `entity_details({connectionId, targets: [{display: "<identifier>"}]})` -
|
|
76
|
+
confirm the identifier resolves; inspect native types, FK/PK, and
|
|
77
|
+
sampleValues.
|
|
78
|
+
3. For literal values from the doc, such as status codes or plan tiers,
|
|
79
|
+
check whether they appear in `entity_details` sampleValues for the relevant
|
|
80
|
+
column. If sampleValues is short or the sample may have missed real values,
|
|
81
|
+
run a `sql_execution` probe with the same warehouse connection id:
|
|
82
|
+
`sql_execution({connectionId, sql: "SELECT DISTINCT <col> FROM <ref> LIMIT 50"})`.
|
|
83
|
+
4. If the candidate identifier still does not resolve, do one of:
|
|
84
|
+
- Use `sql_execution({connectionId, sql: "SELECT 1 FROM <ref> LIMIT 0"})`.
|
|
85
|
+
If it errors, the identifier is fictional.
|
|
86
|
+
- Wrap the identifier in `[unverified - from <rawPath>]` in the wiki body,
|
|
87
|
+
citing the exact raw path that mentioned it.
|
|
88
|
+
- When recording `emit_unmapped_fallback` with `no_physical_table`, include
|
|
89
|
+
the failing probe error in `clarification`.
|
|
90
|
+
5. Never copy `<schema>.<table>` placeholder strings from these instructions
|
|
91
|
+
into output.
|
|
92
|
+
|
|
93
|
+
## Tools
|
|
94
|
+
|
|
95
|
+
Allowed: `read_raw_file`, `read_raw_span`, `wiki_search`, `wiki_read`, `wiki_write`, `discover_data`, `entity_details`, `sql_execution`, `sl_discover`, `sl_read_source`, `context_evidence_search`, `context_evidence_read`, `context_evidence_neighbors`, `emit_unmapped_fallback`, `eviction_list`, `emit_eviction_decision`.
|
|
96
|
+
|
|
97
|
+
Not allowed: `context_candidate_write`, `context_candidate_mark`, `sl_write_source`, `sl_edit_source`, `sl_validate`.
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: sigma_ingest
|
|
3
|
+
description: Extract durable ktx wiki knowledge from staged Sigma data model specs and workbook summaries. Load for WorkUnits with unitKey sigma-data-models or sigma-workbooks.
|
|
4
|
+
callers: [memory_agent]
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
# Sigma Ingest
|
|
8
|
+
|
|
9
|
+
Sigma ingest turns staged data model specs and workbook summaries into durable ktx wiki knowledge. The deterministic `project()` step has already written semantic-layer YAML for all warehouse-table data model elements before this skill runs — do not re-write those SL sources.
|
|
10
|
+
|
|
11
|
+
## Work unit structure
|
|
12
|
+
|
|
13
|
+
Sigma produces at minimum two work units per ingest run:
|
|
14
|
+
|
|
15
|
+
- `sigma-data-models` or `sigma-data-models-N`
|
|
16
|
+
- `rawFiles`: `data-models/<id>.json` files (one per data model in this batch)
|
|
17
|
+
- `peerFileIndex`: `workbooks/<id>.json` files + `sigma-manifest.json` + `sigma-projection-config.json`
|
|
18
|
+
- When the workspace has more than 50 data models, split into batches: `sigma-data-models-0`, `sigma-data-models-1`, … with `displayLabel` like `"Sigma: data models (1/8)"`. When ≤50 data models, the unitKey is simply `sigma-data-models` with no suffix.
|
|
19
|
+
- `sigma-workbooks` or `sigma-workbooks-N`
|
|
20
|
+
- `rawFiles`: `workbooks/<id>.json` files (one per workbook in this batch)
|
|
21
|
+
- `peerFileIndex`: `data-models/<id>.json` files + `sigma-manifest.json` + `sigma-projection-config.json`
|
|
22
|
+
- When the workspace has more than 2000 workbooks, split into batches: `sigma-workbooks-0`, `sigma-workbooks-1`, … with `displayLabel` like `"Sigma: workbooks (1/4)"`. When ≤2000 workbooks, the unitKey is simply `sigma-workbooks` with no suffix.
|
|
23
|
+
|
|
24
|
+
`sigma-manifest.json` and `sigma-projection-config.json` are never in `rawFiles`. They live at the staged dir root and always appear in `peerFileIndex`.
|
|
25
|
+
|
|
26
|
+
## Staged file shapes
|
|
27
|
+
|
|
28
|
+
**`data-models/<id>.json`** — one per data model (in `rawFiles` for data-model units):
|
|
29
|
+
```json
|
|
30
|
+
{
|
|
31
|
+
"sigmaId": "abc-123",
|
|
32
|
+
"name": "Revenue Model",
|
|
33
|
+
"path": "Finance/Revenue Model",
|
|
34
|
+
"latestVersion": 3,
|
|
35
|
+
"updatedAt": "2026-01-15T00:00:00Z",
|
|
36
|
+
"isArchived": false,
|
|
37
|
+
"spec": {
|
|
38
|
+
"name": "Revenue Model",
|
|
39
|
+
"pages": [{
|
|
40
|
+
"id": "p1",
|
|
41
|
+
"name": "Main",
|
|
42
|
+
"elements": [{
|
|
43
|
+
"id": "elem1",
|
|
44
|
+
"kind": "table",
|
|
45
|
+
"name": "Opportunities",
|
|
46
|
+
"hidden": false,
|
|
47
|
+
"source": {
|
|
48
|
+
"kind": "warehouse-table",
|
|
49
|
+
"connectionId": "<sigma-internal-uuid>",
|
|
50
|
+
"path": ["DATABASE", "SCHEMA", "OPPORTUNITIES"]
|
|
51
|
+
},
|
|
52
|
+
"columns": [
|
|
53
|
+
{ "id": "c1", "name": "Deal Amount", "formula": "[OPPORTUNITIES/Amount]", "description": "Net contract value in USD" },
|
|
54
|
+
{ "id": "c2", "name": "Total ARR", "formula": "Sum([OPPORTUNITIES/ARR])", "description": "Annualised recurring revenue" }
|
|
55
|
+
]
|
|
56
|
+
}]
|
|
57
|
+
}]
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
`source.kind` discriminates:
|
|
63
|
+
- `warehouse-table` — element maps directly to a warehouse table. Has `connectionId` and `path` (array of path segments forming the fully-qualified table name). `project()` writes an SL source when `connectionMappings` covers this `connectionId`.
|
|
64
|
+
- `table` — element is a derived view layered on top of another element; identified by `source.elementId`. No warehouse path. Wiki-only.
|
|
65
|
+
|
|
66
|
+
**`workbooks/<id>.json`** — one per workbook, in `rawFiles` for workbook units (summary only; no spec endpoint exists):
|
|
67
|
+
```json
|
|
68
|
+
{
|
|
69
|
+
"sigmaId": "wb-abc",
|
|
70
|
+
"name": "ARR Tracker",
|
|
71
|
+
"path": "Finance/Dashboards",
|
|
72
|
+
"latestVersion": 2,
|
|
73
|
+
"updatedAt": "2026-01-16T00:00:00Z",
|
|
74
|
+
"isArchived": false,
|
|
75
|
+
"workbookUrlId": "57a96EMo3G...",
|
|
76
|
+
"description": "Tracks ARR by segment and cohort for the finance team"
|
|
77
|
+
}
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
**Peer files (available via `peerFileIndex`, not `rawFiles`):**
|
|
81
|
+
|
|
82
|
+
**`sigma-manifest.json`** — fetch summary; use for provenance only.
|
|
83
|
+
|
|
84
|
+
**`sigma-projection-config.json`** — written by `fetch()`, contains two fields the skill must read:
|
|
85
|
+
|
|
86
|
+
- `connectionMappings`: `{sigmaInternalUuid: ktxWarehouseConnectionId}`. Use the mapped warehouse connection ID for `entity_details` when verifying warehouse identifiers found in data model specs.
|
|
87
|
+
- `workbookFilter`: the filter settings that were active when workbooks were last fetched:
|
|
88
|
+
- `includeArchived` (default `false`) — when `false`, archived workbooks are not in `workbooks/`; `isArchived: true` files will only appear when this was `true`.
|
|
89
|
+
- `includeExplorations` (default `false`) — when `false`, exploration-type workbooks (unsaved analyses) are excluded; treat present workbooks as intentional, curated reports.
|
|
90
|
+
- `updatedSince` (optional ISO 8601 string) — when set, only workbooks updated on or after this date are staged; the set is a recent-changes slice, not the full workspace. Do not infer that absent workbooks were deleted.
|
|
91
|
+
|
|
92
|
+
`sigma-manifest.json` also reflects any active `dataModelFilter`. When `dataModelFilter.updatedSince` was set during fetch, `dataModelCount` reflects only matching models, not the full workspace. Do not infer that absent data models were deleted.
|
|
93
|
+
|
|
94
|
+
Read `sigma-projection-config.json` first and keep `workbookFilter` in scope while processing the WorkUnit.
|
|
95
|
+
|
|
96
|
+
## Required workflow
|
|
97
|
+
|
|
98
|
+
1. Read every `rawFiles` entry for the WorkUnit.
|
|
99
|
+
2. Read `sigma-projection-config.json` from the staged dir to get `connectionMappings`.
|
|
100
|
+
3. For each data model file: extract business semantics from element names, column descriptions, and the domain context of the model. Skip hidden elements and hidden columns.
|
|
101
|
+
4. For each workbook file: extract business domain knowledge from the name and description. When `workbookFilter.updatedSince` is set, treat the staged set as a recent-changes slice — absent workbooks were not deleted, they were simply outside the filter window.
|
|
102
|
+
5. Use `discover_data` before writing to find existing wiki pages on the same topic.
|
|
103
|
+
6. Write wiki candidates with `context_candidate_write`. Do not call `wiki_write` directly from a Sigma WorkUnit; Stage 4 reconciliation promotes candidates.
|
|
104
|
+
7. Do not write or edit SL sources. The `project()` step owns all SL output for Sigma.
|
|
105
|
+
|
|
106
|
+
## Identifier Verification Protocol
|
|
107
|
+
|
|
108
|
+
Before writing a wiki page or SL source on any topic:
|
|
109
|
+
|
|
110
|
+
1. `discover_data({query: "<topic>"})` - see what wikis, SL sources, and raw
|
|
111
|
+
tables already exist. Prefer updating existing pages over creating new ones.
|
|
112
|
+
|
|
113
|
+
Before emitting any `schema.table` or `schema.table.column` into a wiki body,
|
|
114
|
+
SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`:
|
|
115
|
+
|
|
116
|
+
2. `entity_details({connectionId, targets: [{display: "<identifier>"}]})` -
|
|
117
|
+
confirm the identifier resolves; inspect native types, FK/PK, and
|
|
118
|
+
sampleValues. Use the warehouse `connectionId` from `connectionMappings` in
|
|
119
|
+
`sigma-projection-config.json`, not the Sigma connection ID. If
|
|
120
|
+
`connectionMappings` has no entry for the element's `source.connectionId`,
|
|
121
|
+
skip `entity_details` — there is no mapped warehouse to verify against —
|
|
122
|
+
and wrap any identifier references with `[unverified - from <rawPath>]`.
|
|
123
|
+
3. For literal values from the source, such as status codes or plan tiers,
|
|
124
|
+
check whether they appear in `entity_details` sampleValues for the relevant
|
|
125
|
+
column. If sampleValues is short or the sample may have missed real values,
|
|
126
|
+
run a `sql_execution` probe with the same warehouse connection id:
|
|
127
|
+
`sql_execution({connectionId, sql: "SELECT DISTINCT <col> FROM <ref> LIMIT 50"})`.
|
|
128
|
+
4. If the candidate identifier still does not resolve, do one of:
|
|
129
|
+
- Use `sql_execution({connectionId, sql: "SELECT 1 FROM <ref> LIMIT 0"})`.
|
|
130
|
+
If it errors, the identifier is fictional.
|
|
131
|
+
- Wrap the identifier in `[unverified - from <rawPath>]` in the wiki body,
|
|
132
|
+
citing the exact raw path that mentioned it.
|
|
133
|
+
- When recording `emit_unmapped_fallback` with `no_physical_table`, include
|
|
134
|
+
the failing probe error in `clarification`.
|
|
135
|
+
5. Never copy `<schema>.<table>` placeholder strings from these instructions
|
|
136
|
+
into output.
|
|
137
|
+
|
|
138
|
+
## Data model elements
|
|
139
|
+
|
|
140
|
+
### Warehouse-table elements (`source.kind === "warehouse-table"`)
|
|
141
|
+
|
|
142
|
+
`project()` writes an SL source for a warehouse-table element **only when** the element's `source.connectionId` has an entry in `connectionMappings`. When no mapping exists, no SL source is written and the element is wiki-only.
|
|
143
|
+
|
|
144
|
+
To determine whether an SL source exists: check whether `connectionMappings[element.source.connectionId]` resolves. If it does, use `sl_discover` to find the source by its slugified name (`<dataModelName>_<elementName>`), then:
|
|
145
|
+
|
|
146
|
+
- Read the existing SL source with `sl_read_source` to understand what columns and measures are captured.
|
|
147
|
+
- Write a wiki candidate about the business domain if the element name, column descriptions, or data model description reveals durable knowledge not already in the wiki.
|
|
148
|
+
- `sl_refs` in the wiki candidate should point to the already-written SL source name.
|
|
149
|
+
|
|
150
|
+
If `connectionMappings` has no entry for the element's `source.connectionId`, treat the element as wiki-only — do not attempt `sl_discover` or `sl_read_source` for it, as no source was written.
|
|
151
|
+
|
|
152
|
+
### Joins within a data model
|
|
153
|
+
|
|
154
|
+
Joins are not projected in v1; `joins: []` is always written by `project()`. `Lookup()` formulas may be described in wiki prose instead.
|
|
155
|
+
|
|
156
|
+
### Non-warehouse elements (`source.kind === "table"`)
|
|
157
|
+
|
|
158
|
+
These reference another element by `elementId` — they are derived views layered on top of a warehouse-table element. They have no warehouse path of their own. Do not attempt SL writes for these elements. They may produce wiki candidates if their column names or descriptions reveal business semantics not captured by the underlying warehouse-table element.
|
|
159
|
+
|
|
160
|
+
## Workbooks
|
|
161
|
+
|
|
162
|
+
Workbooks have summary metadata only. There is no spec endpoint.
|
|
163
|
+
|
|
164
|
+
Extract business domain knowledge from:
|
|
165
|
+
- `name`: the workbook's primary topic (e.g. "ARR Tracker" → ARR tracking concepts)
|
|
166
|
+
- `description`: business context and intended audience
|
|
167
|
+
- `path`: team or functional area (e.g. `Finance/Dashboards`)
|
|
168
|
+
|
|
169
|
+
Write wiki candidates when the name or description reveals a reusable business concept, metric definition, or domain convention. Write one candidate per distinct concept, not one per workbook.
|
|
170
|
+
|
|
171
|
+
Skip workbooks whose name or description contains no durable business semantics (e.g. "Untitled Workbook", "Test Dashboard").
|
|
172
|
+
|
|
173
|
+
## Capture rules
|
|
174
|
+
|
|
175
|
+
Write wiki candidates for:
|
|
176
|
+
- Metric definitions mentioned in element names or column descriptions (e.g. "Net ARR", "Churned MRR")
|
|
177
|
+
- Domain conventions such as cohort definitions, segment taxonomies, or fiscal calendar rules
|
|
178
|
+
- Relationships between business entities revealed by data model joins
|
|
179
|
+
|
|
180
|
+
Skip:
|
|
181
|
+
- Visualization settings, layout, colors, chart types
|
|
182
|
+
- Owner names, folder paths, and version numbers as wiki narrative
|
|
183
|
+
- Hidden elements and hidden columns
|
|
184
|
+
- Data model names that are purely technical with no business meaning
|
|
185
|
+
- When `workbookFilter.includeExplorations` is `false` (the default), all staged workbooks are intentional reports — no extra exploration filter needed. When it is `true`, workbooks without a description or with a generic auto-generated name are likely ephemeral explorations; skip those.
|
|
186
|
+
|
|
187
|
+
## Usage signals
|
|
188
|
+
|
|
189
|
+
Sigma workbooks carry `latestVersion` but no usage counts. Treat a higher `latestVersion` as weak evidence of continued maintenance; do not include version numbers in wiki prose.
|
|
@@ -112,6 +112,30 @@ All three fields use REPLACE semantics on update:
|
|
|
112
112
|
- Pass `[]` → field is cleared.
|
|
113
113
|
- Pass `[values]` → replaces existing with exactly those values (no merging).
|
|
114
114
|
|
|
115
|
+
## Connection scoping
|
|
116
|
+
|
|
117
|
+
A project may have several databases whose schemas reuse the same concept names
|
|
118
|
+
(two warehouses each with `orders`, `customers`, …). The `connections`
|
|
119
|
+
frontmatter field keeps database-specific pages from polluting searches about
|
|
120
|
+
other databases.
|
|
121
|
+
|
|
122
|
+
- The `wiki_write` tool accepts a `connections` field (list of connection ids,
|
|
123
|
+
same REPLACE semantics as `tags`). Absent or empty ⇒ the page is **unscoped**
|
|
124
|
+
and applies to every connection.
|
|
125
|
+
- When this ingest/turn is scoped to a connection (its id appears in the prompt
|
|
126
|
+
context — e.g. `connectionId: warehouse` in the SL Sources header or the
|
|
127
|
+
`<context>` block), set `connections: [<that id>]` on pages whose content is
|
|
128
|
+
**specific to that database** ("in this warehouse `user_id` is the device id,
|
|
129
|
+
not the account id"). Pair this with a connection-distinctive key so two
|
|
130
|
+
databases' same-concept pages can coexist: `orders_sales_db`, not `orders`.
|
|
131
|
+
- Leave `connections` empty for clearly **org-wide** knowledge ("fiscal year
|
|
132
|
+
starts in February") so it stays visible everywhere. Do not scope a page to a
|
|
133
|
+
connection just because the turn happened to be connection-scoped.
|
|
134
|
+
- Keys are still a flat, global namespace; `connections` does not namespace
|
|
135
|
+
them. A connection-scoped write whose key already belongs to a page scoped to
|
|
136
|
+
a *different* connection is rejected to prevent silently overwriting it — pick
|
|
137
|
+
a connection-distinctive key instead.
|
|
138
|
+
|
|
115
139
|
## Editing existing pages
|
|
116
140
|
|
|
117
141
|
Two modes:
|
package/dist/sql.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { FEDERATED_CONNECTION_ID } from './context/connections/federation.js';
|
|
2
2
|
import { executeProjectReadOnlySql } from './context/connections/project-sql-executor.js';
|
|
3
|
+
import { assertSqlQueryableConnection } from './context/connections/dialects.js';
|
|
3
4
|
import { resolveConfiguredConnection } from './context/connections/resolve-connection.js';
|
|
4
5
|
import { loadKtxProject } from './context/project/project.js';
|
|
5
6
|
import { sqlAnalysisDialectForDriver } from './context/sql-analysis/dialect.js';
|
|
@@ -90,6 +91,9 @@ export async function runKtxSql(args, io = process, deps = {}) {
|
|
|
90
91
|
const connection = isFederated ? undefined : resolveConfiguredConnection(project.config, args.connectionId);
|
|
91
92
|
driver = isFederated ? 'duckdb' : String(connection?.driver ?? 'unknown').toLowerCase();
|
|
92
93
|
demoConnection = isFederated ? false : isDemoConnection(args.connectionId, connection);
|
|
94
|
+
if (!isFederated) {
|
|
95
|
+
assertSqlQueryableConnection(args.connectionId, connection?.driver);
|
|
96
|
+
}
|
|
93
97
|
const createSqlAnalysis = deps.createSqlAnalysis ??
|
|
94
98
|
(() => createManagedDaemonSqlAnalysisPort({
|
|
95
99
|
cliVersion: args.cliVersion,
|
package/dist/status-project.d.ts
CHANGED
|
@@ -75,6 +75,10 @@ interface LocalStatsIngestPerConnection {
|
|
|
75
75
|
connectionId: string;
|
|
76
76
|
adapter: string;
|
|
77
77
|
lastCompletedAt: string;
|
|
78
|
+
skippedObjects: Array<{
|
|
79
|
+
name: string;
|
|
80
|
+
reason: string;
|
|
81
|
+
}>;
|
|
78
82
|
}
|
|
79
83
|
interface LocalStatsSemanticLayerEntry {
|
|
80
84
|
connectionId: string;
|
package/dist/status-project.js
CHANGED
|
@@ -4,6 +4,7 @@ import { runClaudeCodeAuthProbe } from './context/llm/claude-code-runtime.js';
|
|
|
4
4
|
import { CODEX_ISOLATION_WARNING, CODEX_ISOLATION_WARNING_FIX, } from './context/llm/codex-isolation.js';
|
|
5
5
|
import { runCodexAuthProbe } from './context/llm/codex-runtime.js';
|
|
6
6
|
import { ktxLocalStateDbPath } from './context/project/local-state-db.js';
|
|
7
|
+
import { listReferencedConnectionIds } from './context/wiki/local-knowledge.js';
|
|
7
8
|
import { isQueryHistoryEnabled, queryHistoryDialectForConnection, } from './context/ingest/adapters/historic-sql/connection-dialect.js';
|
|
8
9
|
import { historicSqlProbeCatalogName, runHistoricSqlReadinessProbe, } from './context/ingest/historic-sql-probes.js';
|
|
9
10
|
import { formatClaudeCodePromptCachingFix, formatClaudeCodePromptCachingWarning, ignoredClaudeCodePromptCachingFields, } from './claude-code-prompt-caching.js';
|
|
@@ -387,6 +388,29 @@ function buildStorageStatus(config) {
|
|
|
387
388
|
gitAuthor: config.storage.git.author,
|
|
388
389
|
};
|
|
389
390
|
}
|
|
391
|
+
/**
|
|
392
|
+
* Warn (never fail) when stored wiki pages reference connection ids that are no
|
|
393
|
+
* longer in `ktx.yaml`. Config and page content evolve independently, so a
|
|
394
|
+
* dangling reference is a soft condition — the pages still load, search, and
|
|
395
|
+
* read; it just signals a typo or a removed connection.
|
|
396
|
+
*/
|
|
397
|
+
async function buildUnknownConnectionWarning(project) {
|
|
398
|
+
let referenced;
|
|
399
|
+
try {
|
|
400
|
+
referenced = await listReferencedConnectionIds(project);
|
|
401
|
+
}
|
|
402
|
+
catch {
|
|
403
|
+
return null;
|
|
404
|
+
}
|
|
405
|
+
const unknown = referenced.filter((id) => !Object.hasOwn(project.config.connections, id));
|
|
406
|
+
if (unknown.length === 0) {
|
|
407
|
+
return null;
|
|
408
|
+
}
|
|
409
|
+
return {
|
|
410
|
+
message: `Wiki pages reference connection id(s) not in ktx.yaml: ${unknown.join(', ')}. Those pages still load and search.`,
|
|
411
|
+
fix: 'Add the connection(s) via `ktx setup`, or update the pages’ `connections` frontmatter.',
|
|
412
|
+
};
|
|
413
|
+
}
|
|
390
414
|
function buildWarnings(config, connections, llm, embeddings) {
|
|
391
415
|
const warnings = [];
|
|
392
416
|
for (const adapter of config.ingest.adapters) {
|
|
@@ -555,6 +579,21 @@ function tryQuery(run, fallback) {
|
|
|
555
579
|
return fallback;
|
|
556
580
|
}
|
|
557
581
|
}
|
|
582
|
+
function skippedObjectsFromReportBody(bodyJson) {
|
|
583
|
+
try {
|
|
584
|
+
const body = JSON.parse(bodyJson);
|
|
585
|
+
const skipped = body.fetch?.skipped;
|
|
586
|
+
if (!Array.isArray(skipped))
|
|
587
|
+
return [];
|
|
588
|
+
return skipped.map((issue) => ({
|
|
589
|
+
name: typeof issue.entityId === 'string' && issue.entityId.length > 0 ? issue.entityId : 'object',
|
|
590
|
+
reason: typeof issue.message === 'string' ? issue.message : 'introspection failed',
|
|
591
|
+
}));
|
|
592
|
+
}
|
|
593
|
+
catch {
|
|
594
|
+
return [];
|
|
595
|
+
}
|
|
596
|
+
}
|
|
558
597
|
/** @internal */
|
|
559
598
|
export async function buildLocalStatsStatus(project) {
|
|
560
599
|
const dbPath = ktxLocalStateDbPath(project);
|
|
@@ -581,8 +620,10 @@ export async function buildLocalStatsStatus(project) {
|
|
|
581
620
|
const totalCompletedRuns = tryQuery(() => db
|
|
582
621
|
.prepare(`SELECT COUNT(*) AS n FROM local_ingest_reports WHERE status = 'done'`)
|
|
583
622
|
.get()?.n ?? 0, 0);
|
|
584
|
-
const ingestRows = tryQuery(() =>
|
|
585
|
-
|
|
623
|
+
const ingestRows = tryQuery(() =>
|
|
624
|
+
// SQLite returns body_json from the MAX(completed_at) row for each group.
|
|
625
|
+
db
|
|
626
|
+
.prepare(`SELECT connection_id, adapter, MAX(completed_at) AS last_completed_at, body_json
|
|
586
627
|
FROM local_ingest_reports
|
|
587
628
|
WHERE status = 'done'
|
|
588
629
|
GROUP BY connection_id, adapter`)
|
|
@@ -595,6 +636,7 @@ export async function buildLocalStatsStatus(project) {
|
|
|
595
636
|
connectionId: row.connection_id,
|
|
596
637
|
adapter: row.adapter,
|
|
597
638
|
lastCompletedAt: row.last_completed_at,
|
|
639
|
+
skippedObjects: skippedObjectsFromReportBody(row.body_json),
|
|
598
640
|
});
|
|
599
641
|
}
|
|
600
642
|
}
|
|
@@ -683,6 +725,10 @@ export async function buildProjectStatus(project, options = {}) {
|
|
|
683
725
|
const queryHistory = await buildQueryHistoryStatus(project, options);
|
|
684
726
|
const pipeline = buildPipelineStatus(config);
|
|
685
727
|
const warnings = buildWarnings(config, connections, llm, embeddings);
|
|
728
|
+
const unknownConnectionWarning = await buildUnknownConnectionWarning(project);
|
|
729
|
+
if (unknownConnectionWarning) {
|
|
730
|
+
warnings.push(unknownConnectionWarning);
|
|
731
|
+
}
|
|
686
732
|
const localStats = await buildLocalStatsStatus(project);
|
|
687
733
|
const { verdict, reason, nextActions } = buildVerdict(llm, embeddings, connections, queryHistory, warnings);
|
|
688
734
|
return {
|
|
@@ -796,6 +842,12 @@ function renderLocalStats(lines, stats, dim, bold) {
|
|
|
796
842
|
const adapterWidth = Math.max(...ingest.perConnection.map((entry) => entry.adapter.length));
|
|
797
843
|
for (const entry of ingest.perConnection) {
|
|
798
844
|
lines.push(` ${entry.connectionId.padEnd(nameWidth)} ${dim(entry.adapter.padEnd(adapterWidth))} ${dim(`last ${formatRelativeFromNow(entry.lastCompletedAt)}`)}`);
|
|
845
|
+
if (entry.skippedObjects.length > 0) {
|
|
846
|
+
const first = entry.skippedObjects[0];
|
|
847
|
+
const extra = entry.skippedObjects.length - 1;
|
|
848
|
+
const detail = `${first.name}: ${first.reason}${extra > 0 ? ` (+${extra} more)` : ''}`;
|
|
849
|
+
lines.push(` ${' '.repeat(nameWidth)} ${dim(`${entry.skippedObjects.length} object${entry.skippedObjects.length === 1 ? '' : 's'} skipped — ${detail}`)}`);
|
|
850
|
+
}
|
|
799
851
|
}
|
|
800
852
|
}
|
|
801
853
|
if (stats.wikiPages.length === 0) {
|
|
@@ -62,10 +62,10 @@ export declare const telemetryEventSchemas: {
|
|
|
62
62
|
isCi: z.ZodBoolean;
|
|
63
63
|
step: z.ZodEnum<{
|
|
64
64
|
embeddings: "embeddings";
|
|
65
|
+
databases: "databases";
|
|
65
66
|
project: "project";
|
|
66
67
|
models: "models";
|
|
67
68
|
sources: "sources";
|
|
68
|
-
databases: "databases";
|
|
69
69
|
context: "context";
|
|
70
70
|
runtime: "runtime";
|
|
71
71
|
secrets: "secrets";
|
|
@@ -254,9 +254,9 @@ export declare const telemetryEventSchemas: {
|
|
|
254
254
|
queryVerb: z.ZodEnum<{
|
|
255
255
|
show: "show";
|
|
256
256
|
with: "with";
|
|
257
|
+
explain: "explain";
|
|
257
258
|
select: "select";
|
|
258
259
|
other: "other";
|
|
259
|
-
explain: "explain";
|
|
260
260
|
}>;
|
|
261
261
|
referencedTableCount: z.ZodNumber;
|
|
262
262
|
durationMs: z.ZodNumber;
|
package/dist/text-ingest.d.ts
CHANGED
|
@@ -2,6 +2,7 @@ import type { MemoryAgentInput } from './context/memory/types.js';
|
|
|
2
2
|
import type { MemoryIngestStatus } from './context/memory/memory-runs.js';
|
|
3
3
|
import { type KtxLocalProject } from './context/project/project.js';
|
|
4
4
|
import type { KtxCliIo } from './cli-runtime.js';
|
|
5
|
+
import { type VerbatimIngestorPort } from './verbatim-ingest.js';
|
|
5
6
|
export interface KtxTextIngestArgs {
|
|
6
7
|
projectDir: string;
|
|
7
8
|
texts: string[];
|
|
@@ -10,6 +11,8 @@ export interface KtxTextIngestArgs {
|
|
|
10
11
|
userId: string;
|
|
11
12
|
json: boolean;
|
|
12
13
|
failFast: boolean;
|
|
14
|
+
/** Code-driven verbatim ingest: store the document body unchanged, LLM derives metadata only. */
|
|
15
|
+
verbatim?: boolean;
|
|
13
16
|
}
|
|
14
17
|
/** @internal */
|
|
15
18
|
export interface TextMemoryIngestPort {
|
|
@@ -24,6 +27,7 @@ export interface KtxTextIngestDeps {
|
|
|
24
27
|
projectDir: string;
|
|
25
28
|
}) => Promise<KtxLocalProject>;
|
|
26
29
|
createMemoryIngest?: (project: KtxLocalProject) => TextMemoryIngestPort;
|
|
30
|
+
createVerbatimIngestor?: (project: KtxLocalProject) => VerbatimIngestorPort;
|
|
27
31
|
readFile?: (path: string) => Promise<string>;
|
|
28
32
|
readStdin?: () => Promise<string>;
|
|
29
33
|
now?: () => number;
|