@kaelio/ktx 0.13.1 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/python/{kaelio_ktx-0.13.1-py3-none-any.whl → kaelio_ktx-0.15.0-py3-none-any.whl} +0 -0
- package/assets/python/manifest.json +4 -4
- package/dist/.tsbuildinfo +1 -1
- package/dist/cli-program.js +1 -1
- package/dist/commands/ingest-commands.d.ts +9 -0
- package/dist/commands/ingest-commands.js +37 -1
- package/dist/commands/knowledge-commands.js +3 -0
- package/dist/commands/setup-commands.js +15 -1
- package/dist/connection-drivers.d.ts +2 -0
- package/dist/connection-drivers.js +7 -3
- package/dist/connection.d.ts +3 -0
- package/dist/connection.js +27 -0
- package/dist/connectors/bigquery/connector.d.ts +17 -6
- package/dist/connectors/bigquery/connector.js +152 -76
- package/dist/connectors/bigquery/dialect.d.ts +2 -2
- package/dist/connectors/clickhouse/connector.d.ts +1 -0
- package/dist/connectors/clickhouse/connector.js +45 -16
- package/dist/connectors/clickhouse/dialect.d.ts +2 -2
- package/dist/connectors/mongodb/connector.d.ts +69 -0
- package/dist/connectors/mongodb/connector.js +307 -0
- package/dist/connectors/mongodb/dialect.d.ts +17 -0
- package/dist/connectors/mongodb/dialect.js +50 -0
- package/dist/connectors/mongodb/live-database-introspection.d.ts +10 -0
- package/dist/connectors/mongodb/live-database-introspection.js +24 -0
- package/dist/connectors/mongodb/schema-inference.d.ts +20 -0
- package/dist/connectors/mongodb/schema-inference.js +110 -0
- package/dist/connectors/mysql/connector.d.ts +1 -0
- package/dist/connectors/mysql/connector.js +18 -2
- package/dist/connectors/mysql/dialect.d.ts +2 -2
- package/dist/connectors/postgres/connector.d.ts +1 -0
- package/dist/connectors/postgres/connector.js +20 -3
- package/dist/connectors/postgres/dialect.d.ts +2 -2
- package/dist/connectors/snowflake/connector.d.ts +1 -0
- package/dist/connectors/snowflake/connector.js +27 -3
- package/dist/connectors/snowflake/dialect.d.ts +2 -2
- package/dist/connectors/sqlite/connector.d.ts +11 -1
- package/dist/connectors/sqlite/connector.js +120 -17
- package/dist/connectors/sqlite/dialect.d.ts +2 -2
- package/dist/connectors/sqlite/read-query-child.d.ts +1 -0
- package/dist/connectors/sqlite/read-query-child.js +23 -0
- package/dist/connectors/sqlserver/connector.d.ts +2 -0
- package/dist/connectors/sqlserver/connector.js +20 -3
- package/dist/connectors/sqlserver/dialect.d.ts +2 -2
- package/dist/context/cache/content-result-cache.d.ts +36 -0
- package/dist/context/cache/content-result-cache.js +14 -0
- package/dist/context/cache/sqlite-content-result-cache.d.ts +18 -0
- package/dist/context/cache/sqlite-content-result-cache.js +223 -0
- package/dist/context/connections/bigquery-identifiers.d.ts +1 -0
- package/dist/context/connections/bigquery-identifiers.js +7 -0
- package/dist/context/connections/configured-connections.d.ts +9 -0
- package/dist/context/connections/configured-connections.js +18 -0
- package/dist/context/connections/dialects.d.ts +30 -4
- package/dist/context/connections/dialects.js +38 -11
- package/dist/context/connections/drivers.js +21 -0
- package/dist/context/connections/gdrive-config.d.ts +19 -0
- package/dist/context/connections/gdrive-config.js +57 -0
- package/dist/context/connections/query-deadline.d.ts +31 -0
- package/dist/context/connections/query-deadline.js +37 -0
- package/dist/context/ingest/adapters/gdrive/chunk.d.ts +3 -0
- package/dist/context/ingest/adapters/gdrive/chunk.js +74 -0
- package/dist/context/ingest/adapters/gdrive/detect.d.ts +1 -0
- package/dist/context/ingest/adapters/gdrive/detect.js +20 -0
- package/dist/context/ingest/adapters/gdrive/fetch.d.ts +6 -0
- package/dist/context/ingest/adapters/gdrive/fetch.js +95 -0
- package/dist/context/ingest/adapters/gdrive/gdrive-client.d.ts +30 -0
- package/dist/context/ingest/adapters/gdrive/gdrive-client.js +132 -0
- package/dist/context/ingest/adapters/gdrive/gdrive.adapter.d.ts +11 -0
- package/dist/context/ingest/adapters/gdrive/gdrive.adapter.js +27 -0
- package/dist/context/ingest/adapters/gdrive/normalize.d.ts +2 -0
- package/dist/context/ingest/adapters/gdrive/normalize.js +256 -0
- package/dist/context/ingest/adapters/gdrive/types.d.ts +153 -0
- package/dist/context/ingest/adapters/gdrive/types.js +36 -0
- package/dist/context/ingest/adapters/live-database/daemon-introspection.js +24 -0
- package/dist/context/ingest/adapters/live-database/fetch-report.d.ts +8 -0
- package/dist/context/ingest/adapters/live-database/fetch-report.js +37 -0
- package/dist/context/ingest/adapters/live-database/live-database.adapter.d.ts +2 -1
- package/dist/context/ingest/adapters/live-database/live-database.adapter.js +9 -2
- package/dist/context/ingest/adapters/live-database/manifest.d.ts +2 -0
- package/dist/context/ingest/adapters/live-database/manifest.js +8 -4
- package/dist/context/ingest/adapters/live-database/scan-outcome.d.ts +17 -0
- package/dist/context/ingest/adapters/live-database/scan-outcome.js +40 -0
- package/dist/context/ingest/adapters/live-database/stage.js +5 -5
- package/dist/context/ingest/adapters/metabase/types.d.ts +1 -1
- package/dist/context/ingest/adapters/metabase/types.js +1 -1
- package/dist/context/ingest/adapters/sigma/chunk.d.ts +6 -0
- package/dist/context/ingest/adapters/sigma/chunk.js +119 -0
- package/dist/context/ingest/adapters/sigma/client-port.d.ts +45 -0
- package/dist/context/ingest/adapters/sigma/client-port.js +1 -0
- package/dist/context/ingest/adapters/sigma/client.d.ts +33 -0
- package/dist/context/ingest/adapters/sigma/client.js +176 -0
- package/dist/context/ingest/adapters/sigma/detect.d.ts +1 -0
- package/dist/context/ingest/adapters/sigma/detect.js +23 -0
- package/dist/context/ingest/adapters/sigma/fetch.d.ts +14 -0
- package/dist/context/ingest/adapters/sigma/fetch.js +191 -0
- package/dist/context/ingest/adapters/sigma/local-sigma.adapter.d.ts +17 -0
- package/dist/context/ingest/adapters/sigma/local-sigma.adapter.js +41 -0
- package/dist/context/ingest/adapters/sigma/project.d.ts +8 -0
- package/dist/context/ingest/adapters/sigma/project.js +186 -0
- package/dist/context/ingest/adapters/sigma/sigma.adapter.d.ts +18 -0
- package/dist/context/ingest/adapters/sigma/sigma.adapter.js +43 -0
- package/dist/context/ingest/adapters/sigma/types.d.ts +80 -0
- package/dist/context/ingest/adapters/sigma/types.js +82 -0
- package/dist/context/ingest/artifact-gates.d.ts +32 -1
- package/dist/context/ingest/artifact-gates.js +85 -18
- package/dist/context/ingest/final-gate-prune.d.ts +35 -0
- package/dist/context/ingest/final-gate-prune.js +229 -0
- package/dist/context/ingest/ingest-bundle.runner.d.ts +3 -0
- package/dist/context/ingest/ingest-bundle.runner.js +459 -240
- package/dist/context/ingest/isolated-diff/patch-integrator.d.ts +0 -11
- package/dist/context/ingest/isolated-diff/patch-integrator.js +0 -44
- package/dist/context/ingest/isolated-diff/work-unit-executor.d.ts +1 -0
- package/dist/context/ingest/isolated-diff/work-unit-executor.js +13 -4
- package/dist/context/ingest/local-adapters.d.ts +2 -1
- package/dist/context/ingest/local-adapters.js +28 -0
- package/dist/context/ingest/local-bundle-runtime.js +7 -2
- package/dist/context/ingest/local-ingest.js +1 -1
- package/dist/context/ingest/local-stage-ingest.d.ts +3 -1
- package/dist/context/ingest/local-stage-ingest.js +3 -1
- package/dist/context/ingest/ports.d.ts +3 -0
- package/dist/context/ingest/report-snapshot.js +13 -3
- package/dist/context/ingest/reports.d.ts +3 -3
- package/dist/context/ingest/reports.js +3 -1
- package/dist/context/ingest/stages/build-wu-context.d.ts +1 -0
- package/dist/context/ingest/stages/build-wu-context.js +2 -1
- package/dist/context/ingest/stages/stage-3-work-units.d.ts +2 -1
- package/dist/context/ingest/stages/stage-3-work-units.js +4 -10
- package/dist/context/ingest/stages/validate-wu-sources.d.ts +12 -0
- package/dist/context/ingest/stages/validate-wu-sources.js +23 -8
- package/dist/context/ingest/tools/read-raw-file.tool.js +10 -5
- package/dist/context/ingest/tools/read-raw-span.tool.js +10 -5
- package/dist/context/ingest/tools/warehouse-verification/discover-data.tool.js +1 -1
- package/dist/context/ingest/tools/warehouse-verification/entity-details.tool.js +1 -1
- package/dist/context/ingest/tools/warehouse-verification/sql-execution.tool.js +1 -1
- package/dist/context/ingest/types.d.ts +3 -0
- package/dist/context/ingest/wiki-body-refs.d.ts +28 -0
- package/dist/context/ingest/wiki-body-refs.js +37 -8
- package/dist/context/ingest/wiki-sl-ref-repair.d.ts +1 -0
- package/dist/context/ingest/wiki-sl-ref-repair.js +4 -0
- package/dist/context/ingest/work-unit-cache.d.ts +67 -0
- package/dist/context/ingest/work-unit-cache.js +230 -0
- package/dist/context/llm/ai-sdk-runtime.d.ts +1 -0
- package/dist/context/llm/ai-sdk-runtime.js +5 -0
- package/dist/context/llm/claude-code-runtime.d.ts +4 -1
- package/dist/context/llm/claude-code-runtime.js +38 -8
- package/dist/context/llm/codex-runtime.d.ts +4 -1
- package/dist/context/llm/codex-runtime.js +48 -35
- package/dist/context/llm/runtime-port.d.ts +26 -0
- package/dist/context/llm/subprocess-generate-object-child.d.ts +1 -0
- package/dist/context/llm/subprocess-generate-object-child.js +34 -0
- package/dist/context/llm/subprocess-generate-object.d.ts +42 -0
- package/dist/context/llm/subprocess-generate-object.js +122 -0
- package/dist/context/mcp/context-tools.d.ts +2 -0
- package/dist/context/mcp/context-tools.js +111 -17
- package/dist/context/mcp/local-project-ports.d.ts +7 -0
- package/dist/context/mcp/local-project-ports.js +29 -0
- package/dist/context/mcp/logger.d.ts +24 -0
- package/dist/context/mcp/logger.js +49 -0
- package/dist/context/mcp/server.js +2 -0
- package/dist/context/mcp/types.d.ts +17 -0
- package/dist/context/memory/memory-agent.service.js +1 -1
- package/dist/context/project/config.d.ts +72 -0
- package/dist/context/project/config.js +5 -0
- package/dist/context/project/driver-schemas.d.ts +35 -0
- package/dist/context/project/driver-schemas.js +89 -1
- package/dist/context/project/project.js +1 -1
- package/dist/context/project/setup-config.js +1 -0
- package/dist/context/scan/description-generation.d.ts +4 -0
- package/dist/context/scan/description-generation.js +100 -13
- package/dist/context/scan/enabled-tables.d.ts +5 -0
- package/dist/context/scan/enabled-tables.js +13 -1
- package/dist/context/scan/enrichment-state.d.ts +50 -7
- package/dist/context/scan/enrichment-state.js +30 -13
- package/dist/context/scan/local-enrichment-artifacts.d.ts +41 -0
- package/dist/context/scan/local-enrichment-artifacts.js +155 -32
- package/dist/context/scan/local-enrichment.d.ts +39 -4
- package/dist/context/scan/local-enrichment.js +345 -92
- package/dist/context/scan/local-scan.d.ts +4 -1
- package/dist/context/scan/local-scan.js +54 -13
- package/dist/context/scan/local-structural-artifacts.d.ts +3 -1
- package/dist/context/scan/local-structural-artifacts.js +5 -0
- package/dist/context/scan/object-introspection.d.ts +21 -0
- package/dist/context/scan/object-introspection.js +35 -0
- package/dist/context/scan/relationship-benchmarks.js +5 -3
- package/dist/context/scan/relationship-composite-candidates.d.ts +6 -3
- package/dist/context/scan/relationship-composite-candidates.js +13 -1
- package/dist/context/scan/relationship-detection-budget.d.ts +35 -0
- package/dist/context/scan/relationship-detection-budget.js +53 -0
- package/dist/context/scan/relationship-diagnostics.d.ts +5 -0
- package/dist/context/scan/relationship-diagnostics.js +2 -0
- package/dist/context/scan/relationship-discovery.d.ts +9 -3
- package/dist/context/scan/relationship-discovery.js +27 -4
- package/dist/context/scan/relationship-llm-proposal.js +36 -27
- package/dist/context/scan/relationship-profiling.d.ts +7 -3
- package/dist/context/scan/relationship-profiling.js +53 -41
- package/dist/context/scan/relationship-validation.d.ts +6 -4
- package/dist/context/scan/relationship-validation.js +46 -32
- package/dist/context/scan/sqlite-local-enrichment-state-store.d.ts +8 -1
- package/dist/context/scan/sqlite-local-enrichment-state-store.js +71 -159
- package/dist/context/scan/types.d.ts +5 -3
- package/dist/context/scan/types.js +2 -0
- package/dist/context/sl/local-query.js +5 -0
- package/dist/context/sl/pglite-sl-search-prototype.js +1 -1
- package/dist/context/sl/semantic-layer.service.js +1 -1
- package/dist/context/sl/source-files.d.ts +5 -0
- package/dist/context/sl/source-files.js +17 -11
- package/dist/context/sl/tools/connection-id-schema.js +1 -1
- package/dist/context/sql-analysis/dialect-notes.d.ts +9 -0
- package/dist/context/sql-analysis/dialect-notes.js +40 -0
- package/dist/context/sql-analysis/dialects/bigquery.md +13 -0
- package/dist/context/sql-analysis/dialects/clickhouse.md +9 -0
- package/dist/context/sql-analysis/dialects/mysql.md +9 -0
- package/dist/context/sql-analysis/dialects/postgres.md +10 -0
- package/dist/context/sql-analysis/dialects/snowflake.md +10 -0
- package/dist/context/sql-analysis/dialects/sqlite.md +11 -0
- package/dist/context/sql-analysis/dialects/tsql.md +10 -0
- package/dist/context/wiki/keys.js +1 -1
- package/dist/context/wiki/knowledge-wiki.service.d.ts +4 -4
- package/dist/context/wiki/knowledge-wiki.service.js +2 -1
- package/dist/context/wiki/local-knowledge.d.ts +13 -0
- package/dist/context/wiki/local-knowledge.js +50 -6
- package/dist/context/wiki/sqlite-knowledge-index.d.ts +2 -0
- package/dist/context/wiki/sqlite-knowledge-index.js +21 -5
- package/dist/context/wiki/tools/wiki-write.tool.d.ts +2 -0
- package/dist/context/wiki/tools/wiki-write.tool.js +27 -0
- package/dist/context/wiki/types.d.ts +6 -0
- package/dist/context-build-view.d.ts +3 -0
- package/dist/context-build-view.js +1 -0
- package/dist/knowledge.d.ts +2 -0
- package/dist/knowledge.js +12 -1
- package/dist/local-adapters.js +11 -0
- package/dist/mcp-http-server.js +15 -1
- package/dist/mcp-server-factory.d.ts +2 -0
- package/dist/mcp-server-factory.js +15 -1
- package/dist/mcp-stdio-server.js +10 -2
- package/dist/notion-page-picker.js +1 -1
- package/dist/prompts/memory_agent_external_ingest.md +2 -0
- package/dist/public-ingest.d.ts +3 -1
- package/dist/public-ingest.js +4 -0
- package/dist/scan.d.ts +3 -1
- package/dist/scan.js +7 -0
- package/dist/setup-databases.d.ts +1 -1
- package/dist/setup-databases.js +43 -1
- package/dist/setup-sources.d.ts +6 -1
- package/dist/setup-sources.js +208 -48
- package/dist/setup.d.ts +3 -0
- package/dist/setup.js +6 -1
- package/dist/skills/analytics/SKILL.md +200 -2
- package/dist/skills/gdrive_synthesize/SKILL.md +97 -0
- package/dist/skills/sigma_ingest/SKILL.md +189 -0
- package/dist/skills/wiki_capture/SKILL.md +24 -0
- package/dist/sql.js +4 -0
- package/dist/status-project.d.ts +4 -0
- package/dist/status-project.js +54 -2
- package/dist/telemetry/events.d.ts +2 -2
- package/dist/text-ingest.d.ts +4 -0
- package/dist/text-ingest.js +58 -29
- package/dist/verbatim-ingest.d.ts +46 -0
- package/dist/verbatim-ingest.js +193 -0
- package/package.json +5 -1
- package/dist/context/ingest/final-gate-repair.d.ts +0 -28
- package/dist/context/ingest/final-gate-repair.js +0 -91
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import type { KtxLlmRuntimePort } from '../../context/llm/runtime-port.js';
|
|
2
2
|
import { type KtxScanRelationshipConfig } from '../project/config.js';
|
|
3
|
-
import { type KtxScanEnrichmentStateStore } from './enrichment-state.js';
|
|
3
|
+
import { type KtxScanEmbeddingIdentity, type KtxScanEnrichmentStateStore, type KtxScanLlmIdentity } from './enrichment-state.js';
|
|
4
|
+
import type { KtxScanDescriptionResumeStore } from './local-enrichment-artifacts.js';
|
|
4
5
|
import type { KtxEmbeddingUpdate, KtxEnrichedSchema, KtxRelationshipUpdate } from './enrichment-types.js';
|
|
5
6
|
import type { KtxCompositeRelationshipCandidate } from './relationship-composite-candidates.js';
|
|
7
|
+
import type { KtxRelationshipDetectionStopReason } from './relationship-detection-budget.js';
|
|
6
8
|
import type { KtxResolvedRelationshipDiscoveryCandidate } from './relationship-graph-resolver.js';
|
|
7
9
|
import type { KtxRelationshipProfileArtifact } from './relationship-profiling.js';
|
|
8
|
-
import type { KtxEmbeddingPort, KtxScanConnector, KtxScanContext, KtxScanEnrichmentStateSummary, KtxScanEnrichmentSummary, KtxScanMode, KtxScanRelationshipSummary, KtxScanWarning, KtxSchemaSnapshot, KtxTableRef } from './types.js';
|
|
10
|
+
import type { KtxEmbeddingPort, KtxScanConnector, KtxScanContext, KtxScanEnrichmentStage, KtxScanEnrichmentStateSummary, KtxScanEnrichmentSummary, KtxScanMode, KtxScanRelationshipSummary, KtxScanWarning, KtxSchemaSnapshot, KtxTableRef } from './types.js';
|
|
9
11
|
export interface KtxLocalScanEnrichmentProviders {
|
|
10
12
|
llmRuntime: KtxLlmRuntimePort;
|
|
11
13
|
embedding?: KtxEmbeddingPort | null;
|
|
@@ -14,15 +16,45 @@ export interface KtxLocalScanEnrichmentInput {
|
|
|
14
16
|
connectionId: string;
|
|
15
17
|
mode: KtxScanMode;
|
|
16
18
|
detectRelationships?: boolean;
|
|
19
|
+
/**
|
|
20
|
+
* Enrichment stages to (re)run this invocation. Undefined runs every eligible
|
|
21
|
+
* stage and respects the completed-stage short-circuit (spec-19 resume). When
|
|
22
|
+
* present, only the named stages run — each force-recomputes (bypassing the
|
|
23
|
+
* short-circuit) while unselected stages are left untouched on disk (D3).
|
|
24
|
+
*/
|
|
25
|
+
stages?: KtxScanEnrichmentStage[];
|
|
17
26
|
connector: KtxScanConnector;
|
|
18
27
|
snapshot?: KtxSchemaSnapshot;
|
|
19
28
|
context: KtxScanContext;
|
|
20
29
|
providers: KtxLocalScanEnrichmentProviders | null;
|
|
21
30
|
stateStore?: KtxScanEnrichmentStateStore | null;
|
|
31
|
+
/**
|
|
32
|
+
* Durable per-batch resume record for the descriptions stage. When present, an
|
|
33
|
+
* interrupted descriptions stage resumes by re-enriching only the tables not
|
|
34
|
+
* already flushed (inputHash-gated). Null/undefined disables incremental flush.
|
|
35
|
+
*/
|
|
36
|
+
descriptionResumeStore?: KtxScanDescriptionResumeStore | null;
|
|
37
|
+
/**
|
|
38
|
+
* Lazily loads the descriptions already persisted in the on-disk _schema, used
|
|
39
|
+
* to feed embeddings + relationships their description context when the
|
|
40
|
+
* descriptions stage does not run this invocation (e.g. `--stages relationships`).
|
|
41
|
+
* Called at most once and only when a downstream stage needs it, so a normal
|
|
42
|
+
* full run never pays the read.
|
|
43
|
+
*/
|
|
44
|
+
loadPriorDescriptions?: (snapshot: KtxSchemaSnapshot) => Promise<KtxLocalScanEnrichmentResult['descriptionUpdates']>;
|
|
22
45
|
syncId?: string;
|
|
23
|
-
|
|
46
|
+
/** Description-LLM identity that keys the descriptions + relationships stage hashes. */
|
|
47
|
+
llmIdentity?: KtxScanLlmIdentity;
|
|
48
|
+
/** Embedding-model identity that keys the embeddings stage hash. */
|
|
49
|
+
embeddingIdentity?: KtxScanEmbeddingIdentity;
|
|
24
50
|
relationshipSettings?: KtxScanRelationshipConfig;
|
|
25
51
|
now?: () => Date;
|
|
52
|
+
/**
|
|
53
|
+
* Invoked once the last non-relationship stage completes and before
|
|
54
|
+
* relationship detection runs, so the descriptions + embeddings reach the
|
|
55
|
+
* queryable layer even if the relationship stage is later interrupted.
|
|
56
|
+
*/
|
|
57
|
+
onCheckpoint?: (checkpoint: KtxLocalScanEnrichmentResult) => Promise<void>;
|
|
26
58
|
}
|
|
27
59
|
export interface KtxLocalScanEnrichmentResult {
|
|
28
60
|
snapshot: KtxSchemaSnapshot;
|
|
@@ -40,7 +72,10 @@ export interface KtxLocalScanEnrichmentResult {
|
|
|
40
72
|
relationshipProfile: KtxRelationshipProfileArtifact | null;
|
|
41
73
|
resolvedRelationships: KtxResolvedRelationshipDiscoveryCandidate[] | null;
|
|
42
74
|
compositeRelationships: KtxCompositeRelationshipCandidate[] | null;
|
|
75
|
+
relationshipPartial: {
|
|
76
|
+
reason: KtxRelationshipDetectionStopReason;
|
|
77
|
+
} | null;
|
|
43
78
|
}
|
|
44
79
|
export declare function createDeterministicLocalScanEnrichmentProviders(): KtxLocalScanEnrichmentProviders;
|
|
45
|
-
export declare function snapshotToKtxEnrichedSchema(snapshot: KtxSchemaSnapshot, embeddingsByColumnId?: ReadonlyMap<string, number[]
|
|
80
|
+
export declare function snapshotToKtxEnrichedSchema(snapshot: KtxSchemaSnapshot, embeddingsByColumnId?: ReadonlyMap<string, number[]>, descriptions?: KtxLocalScanEnrichmentResult['descriptionUpdates']): KtxEnrichedSchema;
|
|
46
81
|
export declare function runLocalScanEnrichment(input: KtxLocalScanEnrichmentInput): Promise<KtxLocalScanEnrichmentResult>;
|