@kaelio/ktx 0.13.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/python/{kaelio_ktx-0.13.0-py3-none-any.whl → kaelio_ktx-0.14.0-py3-none-any.whl} +0 -0
- package/assets/python/manifest.json +4 -4
- package/dist/.tsbuildinfo +1 -1
- package/dist/cli-program.js +1 -1
- package/dist/commands/ingest-commands.d.ts +9 -0
- package/dist/commands/ingest-commands.js +37 -1
- package/dist/commands/knowledge-commands.js +3 -0
- package/dist/commands/setup-commands.js +14 -1
- package/dist/connection-drivers.d.ts +2 -0
- package/dist/connection-drivers.js +7 -3
- package/dist/connection.d.ts +3 -0
- package/dist/connection.js +27 -0
- package/dist/connectors/bigquery/connector.d.ts +17 -6
- package/dist/connectors/bigquery/connector.js +152 -76
- package/dist/connectors/bigquery/dialect.d.ts +2 -2
- package/dist/connectors/clickhouse/connector.d.ts +1 -0
- package/dist/connectors/clickhouse/connector.js +45 -16
- package/dist/connectors/clickhouse/dialect.d.ts +2 -2
- package/dist/connectors/mongodb/connector.d.ts +69 -0
- package/dist/connectors/mongodb/connector.js +307 -0
- package/dist/connectors/mongodb/dialect.d.ts +17 -0
- package/dist/connectors/mongodb/dialect.js +50 -0
- package/dist/connectors/mongodb/live-database-introspection.d.ts +10 -0
- package/dist/connectors/mongodb/live-database-introspection.js +24 -0
- package/dist/connectors/mongodb/schema-inference.d.ts +20 -0
- package/dist/connectors/mongodb/schema-inference.js +110 -0
- package/dist/connectors/mysql/connector.d.ts +1 -0
- package/dist/connectors/mysql/connector.js +18 -2
- package/dist/connectors/mysql/dialect.d.ts +2 -2
- package/dist/connectors/postgres/connector.d.ts +1 -0
- package/dist/connectors/postgres/connector.js +20 -3
- package/dist/connectors/postgres/dialect.d.ts +2 -2
- package/dist/connectors/snowflake/connector.d.ts +1 -0
- package/dist/connectors/snowflake/connector.js +27 -3
- package/dist/connectors/snowflake/dialect.d.ts +2 -2
- package/dist/connectors/sqlite/connector.d.ts +11 -1
- package/dist/connectors/sqlite/connector.js +120 -17
- package/dist/connectors/sqlite/dialect.d.ts +2 -2
- package/dist/connectors/sqlite/read-query-child.d.ts +1 -0
- package/dist/connectors/sqlite/read-query-child.js +23 -0
- package/dist/connectors/sqlserver/connector.d.ts +2 -0
- package/dist/connectors/sqlserver/connector.js +23 -5
- package/dist/connectors/sqlserver/dialect.d.ts +2 -2
- package/dist/context/cache/content-result-cache.d.ts +36 -0
- package/dist/context/cache/content-result-cache.js +14 -0
- package/dist/context/cache/sqlite-content-result-cache.d.ts +18 -0
- package/dist/context/cache/sqlite-content-result-cache.js +223 -0
- package/dist/context/connections/bigquery-identifiers.d.ts +1 -0
- package/dist/context/connections/bigquery-identifiers.js +7 -0
- package/dist/context/connections/configured-connections.d.ts +9 -0
- package/dist/context/connections/configured-connections.js +18 -0
- package/dist/context/connections/dialects.d.ts +30 -4
- package/dist/context/connections/dialects.js +38 -11
- package/dist/context/connections/drivers.js +21 -0
- package/dist/context/connections/gdrive-config.d.ts +19 -0
- package/dist/context/connections/gdrive-config.js +57 -0
- package/dist/context/connections/query-deadline.d.ts +31 -0
- package/dist/context/connections/query-deadline.js +37 -0
- package/dist/context/connections/read-only-sql.d.ts +5 -0
- package/dist/context/connections/read-only-sql.js +139 -1
- package/dist/context/ingest/adapters/gdrive/chunk.d.ts +3 -0
- package/dist/context/ingest/adapters/gdrive/chunk.js +74 -0
- package/dist/context/ingest/adapters/gdrive/detect.d.ts +1 -0
- package/dist/context/ingest/adapters/gdrive/detect.js +20 -0
- package/dist/context/ingest/adapters/gdrive/fetch.d.ts +6 -0
- package/dist/context/ingest/adapters/gdrive/fetch.js +95 -0
- package/dist/context/ingest/adapters/gdrive/gdrive-client.d.ts +30 -0
- package/dist/context/ingest/adapters/gdrive/gdrive-client.js +132 -0
- package/dist/context/ingest/adapters/gdrive/gdrive.adapter.d.ts +11 -0
- package/dist/context/ingest/adapters/gdrive/gdrive.adapter.js +27 -0
- package/dist/context/ingest/adapters/gdrive/normalize.d.ts +2 -0
- package/dist/context/ingest/adapters/gdrive/normalize.js +256 -0
- package/dist/context/ingest/adapters/gdrive/types.d.ts +153 -0
- package/dist/context/ingest/adapters/gdrive/types.js +36 -0
- package/dist/context/ingest/adapters/live-database/daemon-introspection.js +24 -0
- package/dist/context/ingest/adapters/live-database/fetch-report.d.ts +8 -0
- package/dist/context/ingest/adapters/live-database/fetch-report.js +37 -0
- package/dist/context/ingest/adapters/live-database/live-database.adapter.d.ts +2 -1
- package/dist/context/ingest/adapters/live-database/live-database.adapter.js +9 -2
- package/dist/context/ingest/adapters/live-database/manifest.d.ts +2 -0
- package/dist/context/ingest/adapters/live-database/manifest.js +8 -4
- package/dist/context/ingest/adapters/live-database/scan-outcome.d.ts +17 -0
- package/dist/context/ingest/adapters/live-database/scan-outcome.js +40 -0
- package/dist/context/ingest/adapters/live-database/stage.js +5 -5
- package/dist/context/ingest/adapters/metabase/types.d.ts +1 -1
- package/dist/context/ingest/adapters/metabase/types.js +1 -1
- package/dist/context/ingest/artifact-gates.d.ts +32 -1
- package/dist/context/ingest/artifact-gates.js +85 -18
- package/dist/context/ingest/final-gate-prune.d.ts +35 -0
- package/dist/context/ingest/final-gate-prune.js +229 -0
- package/dist/context/ingest/ingest-bundle.runner.d.ts +3 -0
- package/dist/context/ingest/ingest-bundle.runner.js +459 -240
- package/dist/context/ingest/isolated-diff/patch-integrator.d.ts +0 -11
- package/dist/context/ingest/isolated-diff/patch-integrator.js +0 -44
- package/dist/context/ingest/isolated-diff/work-unit-executor.d.ts +1 -0
- package/dist/context/ingest/isolated-diff/work-unit-executor.js +13 -4
- package/dist/context/ingest/local-adapters.js +6 -0
- package/dist/context/ingest/local-bundle-runtime.js +7 -2
- package/dist/context/ingest/local-ingest.js +1 -1
- package/dist/context/ingest/local-stage-ingest.d.ts +3 -1
- package/dist/context/ingest/local-stage-ingest.js +3 -1
- package/dist/context/ingest/ports.d.ts +3 -0
- package/dist/context/ingest/report-snapshot.js +13 -3
- package/dist/context/ingest/reports.d.ts +3 -3
- package/dist/context/ingest/reports.js +3 -1
- package/dist/context/ingest/stages/build-wu-context.d.ts +1 -0
- package/dist/context/ingest/stages/build-wu-context.js +2 -1
- package/dist/context/ingest/stages/stage-3-work-units.d.ts +2 -1
- package/dist/context/ingest/stages/stage-3-work-units.js +4 -10
- package/dist/context/ingest/stages/validate-wu-sources.d.ts +12 -0
- package/dist/context/ingest/stages/validate-wu-sources.js +23 -8
- package/dist/context/ingest/tools/read-raw-file.tool.js +10 -5
- package/dist/context/ingest/tools/read-raw-span.tool.js +10 -5
- package/dist/context/ingest/tools/warehouse-verification/discover-data.tool.js +1 -1
- package/dist/context/ingest/tools/warehouse-verification/entity-details.tool.js +1 -1
- package/dist/context/ingest/tools/warehouse-verification/sql-execution.tool.js +1 -1
- package/dist/context/ingest/types.d.ts +3 -0
- package/dist/context/ingest/wiki-body-refs.d.ts +28 -0
- package/dist/context/ingest/wiki-body-refs.js +37 -8
- package/dist/context/ingest/wiki-sl-ref-repair.d.ts +1 -0
- package/dist/context/ingest/wiki-sl-ref-repair.js +4 -0
- package/dist/context/ingest/work-unit-cache.d.ts +67 -0
- package/dist/context/ingest/work-unit-cache.js +230 -0
- package/dist/context/llm/ai-sdk-runtime.d.ts +1 -0
- package/dist/context/llm/ai-sdk-runtime.js +5 -0
- package/dist/context/llm/claude-code-runtime.d.ts +4 -1
- package/dist/context/llm/claude-code-runtime.js +38 -8
- package/dist/context/llm/codex-runtime.d.ts +4 -1
- package/dist/context/llm/codex-runtime.js +48 -35
- package/dist/context/llm/runtime-port.d.ts +26 -0
- package/dist/context/llm/subprocess-generate-object-child.d.ts +1 -0
- package/dist/context/llm/subprocess-generate-object-child.js +34 -0
- package/dist/context/llm/subprocess-generate-object.d.ts +42 -0
- package/dist/context/llm/subprocess-generate-object.js +122 -0
- package/dist/context/mcp/context-tools.d.ts +2 -0
- package/dist/context/mcp/context-tools.js +111 -17
- package/dist/context/mcp/local-project-ports.d.ts +7 -0
- package/dist/context/mcp/local-project-ports.js +29 -0
- package/dist/context/mcp/logger.d.ts +24 -0
- package/dist/context/mcp/logger.js +49 -0
- package/dist/context/mcp/server.js +2 -0
- package/dist/context/mcp/types.d.ts +17 -0
- package/dist/context/memory/memory-agent.service.js +1 -1
- package/dist/context/project/config.d.ts +42 -0
- package/dist/context/project/config.js +5 -0
- package/dist/context/project/driver-schemas.d.ts +20 -0
- package/dist/context/project/driver-schemas.js +50 -1
- package/dist/context/project/project.js +1 -1
- package/dist/context/project/setup-config.js +1 -0
- package/dist/context/scan/description-generation.d.ts +4 -0
- package/dist/context/scan/description-generation.js +100 -13
- package/dist/context/scan/enabled-tables.d.ts +5 -0
- package/dist/context/scan/enabled-tables.js +13 -1
- package/dist/context/scan/enrichment-state.d.ts +50 -7
- package/dist/context/scan/enrichment-state.js +30 -13
- package/dist/context/scan/local-enrichment-artifacts.d.ts +41 -0
- package/dist/context/scan/local-enrichment-artifacts.js +155 -32
- package/dist/context/scan/local-enrichment.d.ts +39 -4
- package/dist/context/scan/local-enrichment.js +345 -92
- package/dist/context/scan/local-scan.d.ts +4 -1
- package/dist/context/scan/local-scan.js +54 -13
- package/dist/context/scan/local-structural-artifacts.d.ts +3 -1
- package/dist/context/scan/local-structural-artifacts.js +5 -0
- package/dist/context/scan/object-introspection.d.ts +21 -0
- package/dist/context/scan/object-introspection.js +35 -0
- package/dist/context/scan/relationship-benchmarks.js +5 -3
- package/dist/context/scan/relationship-composite-candidates.d.ts +6 -3
- package/dist/context/scan/relationship-composite-candidates.js +13 -1
- package/dist/context/scan/relationship-detection-budget.d.ts +35 -0
- package/dist/context/scan/relationship-detection-budget.js +53 -0
- package/dist/context/scan/relationship-diagnostics.d.ts +5 -0
- package/dist/context/scan/relationship-diagnostics.js +2 -0
- package/dist/context/scan/relationship-discovery.d.ts +9 -3
- package/dist/context/scan/relationship-discovery.js +27 -4
- package/dist/context/scan/relationship-llm-proposal.js +36 -27
- package/dist/context/scan/relationship-profiling.d.ts +7 -3
- package/dist/context/scan/relationship-profiling.js +53 -41
- package/dist/context/scan/relationship-validation.d.ts +6 -4
- package/dist/context/scan/relationship-validation.js +46 -32
- package/dist/context/scan/sqlite-local-enrichment-state-store.d.ts +8 -1
- package/dist/context/scan/sqlite-local-enrichment-state-store.js +71 -159
- package/dist/context/scan/types.d.ts +5 -3
- package/dist/context/scan/types.js +2 -0
- package/dist/context/sl/local-query.js +5 -0
- package/dist/context/sl/pglite-sl-search-prototype.js +1 -1
- package/dist/context/sl/semantic-layer.service.js +1 -1
- package/dist/context/sl/source-files.d.ts +5 -0
- package/dist/context/sl/source-files.js +17 -11
- package/dist/context/sl/tools/connection-id-schema.js +1 -1
- package/dist/context/sql-analysis/dialect-notes.d.ts +9 -0
- package/dist/context/sql-analysis/dialect-notes.js +40 -0
- package/dist/context/sql-analysis/dialects/bigquery.md +13 -0
- package/dist/context/sql-analysis/dialects/clickhouse.md +9 -0
- package/dist/context/sql-analysis/dialects/mysql.md +9 -0
- package/dist/context/sql-analysis/dialects/postgres.md +10 -0
- package/dist/context/sql-analysis/dialects/snowflake.md +10 -0
- package/dist/context/sql-analysis/dialects/sqlite.md +11 -0
- package/dist/context/sql-analysis/dialects/tsql.md +10 -0
- package/dist/context/wiki/keys.js +1 -1
- package/dist/context/wiki/knowledge-wiki.service.d.ts +4 -4
- package/dist/context/wiki/knowledge-wiki.service.js +2 -1
- package/dist/context/wiki/local-knowledge.d.ts +13 -0
- package/dist/context/wiki/local-knowledge.js +50 -6
- package/dist/context/wiki/sqlite-knowledge-index.d.ts +2 -0
- package/dist/context/wiki/sqlite-knowledge-index.js +21 -5
- package/dist/context/wiki/tools/wiki-write.tool.d.ts +2 -0
- package/dist/context/wiki/tools/wiki-write.tool.js +27 -0
- package/dist/context/wiki/types.d.ts +6 -0
- package/dist/context-build-view.d.ts +3 -0
- package/dist/context-build-view.js +1 -0
- package/dist/knowledge.d.ts +2 -0
- package/dist/knowledge.js +12 -1
- package/dist/local-adapters.js +11 -0
- package/dist/mcp-http-server.js +15 -1
- package/dist/mcp-server-factory.d.ts +2 -0
- package/dist/mcp-server-factory.js +15 -1
- package/dist/mcp-stdio-server.js +10 -2
- package/dist/notion-page-picker.js +1 -1
- package/dist/prompts/memory_agent_external_ingest.md +2 -0
- package/dist/public-ingest.d.ts +3 -1
- package/dist/public-ingest.js +3 -0
- package/dist/scan.d.ts +3 -1
- package/dist/scan.js +7 -0
- package/dist/setup-databases.d.ts +1 -1
- package/dist/setup-databases.js +43 -1
- package/dist/setup-sources.d.ts +5 -1
- package/dist/setup-sources.js +124 -48
- package/dist/setup.d.ts +3 -0
- package/dist/setup.js +6 -1
- package/dist/skills/analytics/SKILL.md +200 -2
- package/dist/skills/gdrive_synthesize/SKILL.md +97 -0
- package/dist/skills/wiki_capture/SKILL.md +24 -0
- package/dist/sql.js +4 -0
- package/dist/status-project.d.ts +4 -0
- package/dist/status-project.js +54 -2
- package/dist/telemetry/events.d.ts +2 -2
- package/dist/text-ingest.d.ts +4 -0
- package/dist/text-ingest.js +58 -29
- package/dist/verbatim-ingest.d.ts +46 -0
- package/dist/verbatim-ingest.js +193 -0
- package/package.json +5 -1
- package/dist/context/ingest/final-gate-repair.d.ts +0 -28
- package/dist/context/ingest/final-gate-repair.js +0 -91
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { KtxExpectedError } from '../../../../errors.js';
|
|
2
|
+
import { tableRefFromKey } from '../../../scan/table-ref.js';
|
|
3
|
+
const OBJECT_SKIP_CODE = 'object_introspection_failed';
|
|
4
|
+
function formatScopeEntry(key) {
|
|
5
|
+
const ref = tableRefFromKey(key);
|
|
6
|
+
return [ref.catalog, ref.db, ref.name].filter((part) => Boolean(part)).join('.');
|
|
7
|
+
}
|
|
8
|
+
function discoveredObjectNames(snapshot) {
|
|
9
|
+
const raw = snapshot.metadata.discovered_object_names;
|
|
10
|
+
return Array.isArray(raw) ? raw.filter((value) => typeof value === 'string') : [];
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Enforces the partial-vs-total outcome rules for a live-database snapshot,
|
|
14
|
+
* uniformly for every connector. Outcomes follow from object counts, not a
|
|
15
|
+
* mode: a connection with at least one ingested object succeeds (any broken
|
|
16
|
+
* objects ride along as warnings); a connection where every introspected object
|
|
17
|
+
* failed, or a non-empty enabled_tables scope that matched nothing, raises a
|
|
18
|
+
* clear connection error instead of staging an empty layer that would later
|
|
19
|
+
* surface as the generic "did not recognize" message. A legitimately empty
|
|
20
|
+
* database (no scope, no objects) succeeds with an empty layer.
|
|
21
|
+
*/
|
|
22
|
+
export function assertLiveDatabaseScanOutcome(input) {
|
|
23
|
+
const { connectionId, scope, snapshot } = input;
|
|
24
|
+
if (snapshot.tables.length > 0) {
|
|
25
|
+
return;
|
|
26
|
+
}
|
|
27
|
+
const skipped = (snapshot.warnings ?? []).filter((warning) => warning.code === OBJECT_SKIP_CODE);
|
|
28
|
+
if (skipped.length > 0) {
|
|
29
|
+
const detail = skipped.map((warning) => `${warning.table ?? 'object'}: ${warning.message}`).join('; ');
|
|
30
|
+
throw new KtxExpectedError(`Connection "${connectionId}" produced no semantic layer: all ${skipped.length} introspected ` +
|
|
31
|
+
`${skipped.length === 1 ? 'object' : 'objects'} failed (${detail}).`);
|
|
32
|
+
}
|
|
33
|
+
if (scope && scope.size > 0) {
|
|
34
|
+
const requested = [...scope].map(formatScopeEntry).sort();
|
|
35
|
+
const available = discoveredObjectNames(snapshot);
|
|
36
|
+
const availableClause = available.length > 0 ? ` Available objects: ${available.join(', ')}.` : '';
|
|
37
|
+
throw new KtxExpectedError(`enabled_tables for connection "${connectionId}" matched no objects ` +
|
|
38
|
+
`(looked for: ${requested.join(', ')}).${availableClause}`);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
@@ -101,13 +101,13 @@ export async function readLiveDatabaseTableFiles(stagedDir) {
|
|
|
101
101
|
return out;
|
|
102
102
|
}
|
|
103
103
|
export async function detectLiveDatabaseStagedDir(stagedDir) {
|
|
104
|
+
// A valid live-database staging is identified by its connection.json marker.
|
|
105
|
+
// An empty table set is a legitimate outcome (an empty database), so the
|
|
106
|
+
// presence of table files is not required — the total-vs-partial decision is
|
|
107
|
+
// made earlier by assertLiveDatabaseScanOutcome, before staging.
|
|
104
108
|
try {
|
|
105
109
|
const meta = JSON.parse(await readFile(join(stagedDir, LIVE_DATABASE_META_FILE), 'utf8'));
|
|
106
|
-
|
|
107
|
-
return false;
|
|
108
|
-
}
|
|
109
|
-
const files = await readLiveDatabaseTableFiles(stagedDir);
|
|
110
|
-
return files.length > 0;
|
|
110
|
+
return Boolean(meta) && typeof meta === 'object' && !Array.isArray(meta);
|
|
111
111
|
}
|
|
112
112
|
catch {
|
|
113
113
|
return false;
|
|
@@ -118,8 +118,8 @@ export declare const stagedSyncConfigSchema: z.ZodObject<{
|
|
|
118
118
|
}>;
|
|
119
119
|
selections: z.ZodArray<z.ZodObject<{
|
|
120
120
|
selectionType: z.ZodEnum<{
|
|
121
|
-
collection: "collection";
|
|
122
121
|
item: "item";
|
|
122
|
+
collection: "collection";
|
|
123
123
|
}>;
|
|
124
124
|
metabaseObjectId: z.ZodNumber;
|
|
125
125
|
}, z.core.$strip>>;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
const metabaseSyncModeSchema = z.enum(['ALL', 'ONLY', 'EXCEPT']);
|
|
3
|
-
const metabaseLocalConnectionIdSchema = z.string().regex(/^[a-zA-Z0-
|
|
3
|
+
const metabaseLocalConnectionIdSchema = z.string().regex(/^[a-zA-Z0-9_][a-zA-Z0-9_-]*$/);
|
|
4
4
|
/**
|
|
5
5
|
* The lean config the adapter needs at `fetch()` time. Lives in the BullMQ payload's
|
|
6
6
|
* `bundleRef.config` when the runner invokes the adapter. Never persisted beyond one
|
|
@@ -2,6 +2,7 @@ import type { SemanticLayerService } from '../../context/sl/semantic-layer.servi
|
|
|
2
2
|
import type { TouchedSlSource } from '../../context/tools/touched-sl-sources.js';
|
|
3
3
|
import type { KnowledgeWikiService } from '../../context/wiki/knowledge-wiki.service.js';
|
|
4
4
|
import type { WuValidationResult } from './stages/validate-wu-sources.js';
|
|
5
|
+
import { type WikiBodyRefIssue } from './wiki-body-refs.js';
|
|
5
6
|
export interface FinalArtifactGateInput {
|
|
6
7
|
connectionIds: string[];
|
|
7
8
|
changedWikiPageKeys: string[];
|
|
@@ -18,5 +19,35 @@ export interface ProvenanceRawPathValidationInput {
|
|
|
18
19
|
currentRawPaths: Set<string>;
|
|
19
20
|
deletedRawPaths: Set<string>;
|
|
20
21
|
}
|
|
21
|
-
export
|
|
22
|
+
export type FinalArtifactGateFinding = {
|
|
23
|
+
kind: 'invalid_source';
|
|
24
|
+
connectionId: string;
|
|
25
|
+
sourceName: string;
|
|
26
|
+
errors: string[];
|
|
27
|
+
} | {
|
|
28
|
+
kind: 'missing_join_target';
|
|
29
|
+
ownerConnectionId: string;
|
|
30
|
+
ownerSourceName: string;
|
|
31
|
+
targetSourceName: string;
|
|
32
|
+
message: string;
|
|
33
|
+
} | {
|
|
34
|
+
kind: 'missing_wiki_ref';
|
|
35
|
+
pageKey: string;
|
|
36
|
+
targetPageKey: string;
|
|
37
|
+
message: string;
|
|
38
|
+
} | {
|
|
39
|
+
kind: 'missing_wiki_sl_ref';
|
|
40
|
+
pageKey: string;
|
|
41
|
+
ref: string;
|
|
42
|
+
sourceName: string;
|
|
43
|
+
entityName: string | null;
|
|
44
|
+
message: string;
|
|
45
|
+
} | WikiBodyRefIssue;
|
|
46
|
+
export interface FinalArtifactGateResult {
|
|
47
|
+
ok: boolean;
|
|
48
|
+
findings: FinalArtifactGateFinding[];
|
|
49
|
+
}
|
|
50
|
+
export declare function formatFinalArtifactGateFindings(findings: FinalArtifactGateFinding[]): string;
|
|
51
|
+
export declare function isFinalArtifactGateFindingPruneable(finding: FinalArtifactGateFinding): boolean;
|
|
52
|
+
export declare function validateFinalIngestArtifacts(input: FinalArtifactGateInput): Promise<FinalArtifactGateResult>;
|
|
22
53
|
export declare function validateProvenanceRawPaths(input: ProvenanceRawPathValidationInput): void;
|
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
import { findMissingWikiRefs } from '../wiki/wiki-ref-validation.js';
|
|
2
|
-
import {
|
|
2
|
+
import { findInvalidWikiBodyRefIssues } from './wiki-body-refs.js';
|
|
3
|
+
function normalizeRawPath(path) {
|
|
4
|
+
return path.replace(/\\/g, '/').replace(/^\/+/, '');
|
|
5
|
+
}
|
|
3
6
|
function parseSlRef(ref) {
|
|
4
7
|
const withoutConnection = ref.includes('/') ? ref.slice(ref.indexOf('/') + 1) : ref;
|
|
5
8
|
const connectionId = ref.includes('/') ? ref.slice(0, ref.indexOf('/')) : null;
|
|
@@ -14,7 +17,7 @@ function slEntityNames(source) {
|
|
|
14
17
|
]);
|
|
15
18
|
}
|
|
16
19
|
async function validateWikiSlRefs(input) {
|
|
17
|
-
const
|
|
20
|
+
const findings = [];
|
|
18
21
|
const sourcesByConnection = new Map();
|
|
19
22
|
for (const connectionId of input.connectionIds) {
|
|
20
23
|
const { sources } = await input.semanticLayerService.loadAllSources(connectionId);
|
|
@@ -36,18 +39,32 @@ async function validateWikiSlRefs(input) {
|
|
|
36
39
|
}
|
|
37
40
|
}
|
|
38
41
|
if (!source) {
|
|
39
|
-
|
|
42
|
+
findings.push({
|
|
43
|
+
kind: 'missing_wiki_sl_ref',
|
|
44
|
+
pageKey,
|
|
45
|
+
ref,
|
|
46
|
+
sourceName: parsed.sourceName,
|
|
47
|
+
entityName: parsed.entityName,
|
|
48
|
+
message: `${pageKey}: unknown sl_refs entry ${ref}`,
|
|
49
|
+
});
|
|
40
50
|
continue;
|
|
41
51
|
}
|
|
42
52
|
if (parsed.entityName && !slEntityNames(source).has(parsed.entityName)) {
|
|
43
|
-
|
|
53
|
+
findings.push({
|
|
54
|
+
kind: 'missing_wiki_sl_ref',
|
|
55
|
+
pageKey,
|
|
56
|
+
ref,
|
|
57
|
+
sourceName: parsed.sourceName,
|
|
58
|
+
entityName: parsed.entityName,
|
|
59
|
+
message: `${pageKey}: unknown sl_refs entity ${ref}`,
|
|
60
|
+
});
|
|
44
61
|
}
|
|
45
62
|
}
|
|
46
63
|
}
|
|
47
|
-
return
|
|
64
|
+
return findings;
|
|
48
65
|
}
|
|
49
66
|
async function validateWikiRefs(input) {
|
|
50
|
-
const
|
|
67
|
+
const findings = [];
|
|
51
68
|
for (const pageKey of input.changedWikiPageKeys) {
|
|
52
69
|
const page = await input.wikiService.readPage('GLOBAL', null, pageKey);
|
|
53
70
|
if (!page) {
|
|
@@ -62,28 +79,77 @@ async function validateWikiRefs(input) {
|
|
|
62
79
|
content: page.content,
|
|
63
80
|
});
|
|
64
81
|
for (const missingRef of missingRefs) {
|
|
65
|
-
|
|
82
|
+
findings.push({
|
|
83
|
+
kind: 'missing_wiki_ref',
|
|
84
|
+
pageKey,
|
|
85
|
+
targetPageKey: missingRef,
|
|
86
|
+
message: `${pageKey} -> ${missingRef}`,
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
return findings;
|
|
91
|
+
}
|
|
92
|
+
export function formatFinalArtifactGateFindings(findings) {
|
|
93
|
+
const errors = findings.map((finding) => {
|
|
94
|
+
if (finding.kind === 'invalid_source') {
|
|
95
|
+
return `semantic-layer validation failed for ${finding.connectionId}:${finding.sourceName}: ${finding.errors.join('; ')}`;
|
|
96
|
+
}
|
|
97
|
+
if (finding.kind === 'missing_wiki_ref') {
|
|
98
|
+
return `wiki reference targets missing page: ${finding.message}`;
|
|
99
|
+
}
|
|
100
|
+
return finding.message;
|
|
101
|
+
});
|
|
102
|
+
return `final artifact gates failed:\n${errors.join('\n')}`;
|
|
103
|
+
}
|
|
104
|
+
export function isFinalArtifactGateFindingPruneable(finding) {
|
|
105
|
+
switch (finding.kind) {
|
|
106
|
+
case 'invalid_source':
|
|
107
|
+
case 'missing_join_target':
|
|
108
|
+
case 'missing_wiki_ref':
|
|
109
|
+
case 'missing_wiki_sl_ref':
|
|
110
|
+
case 'missing_wiki_body_sl_entity':
|
|
111
|
+
case 'missing_wiki_body_sl_source':
|
|
112
|
+
case 'missing_wiki_body_table':
|
|
113
|
+
return true;
|
|
114
|
+
default: {
|
|
115
|
+
const exhaustive = finding;
|
|
116
|
+
return exhaustive;
|
|
66
117
|
}
|
|
67
118
|
}
|
|
68
|
-
return dangling;
|
|
69
119
|
}
|
|
70
120
|
export async function validateFinalIngestArtifacts(input) {
|
|
71
121
|
// Join-neighbor expansion happens inside validateTouchedSources so work-unit
|
|
72
122
|
// validation and this gate check the same set — a source that passes one
|
|
73
123
|
// passes the other.
|
|
74
124
|
const validation = await input.validateTouchedSources(input.touchedSlSources);
|
|
75
|
-
const
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
125
|
+
const findings = [];
|
|
126
|
+
for (const invalid of validation.invalidSources) {
|
|
127
|
+
const [connectionId = '', sourceName = ''] = invalid.source.split(':', 2);
|
|
128
|
+
const issues = invalid.issues ?? invalid.errors.map((message) => ({ kind: 'source_validation', message }));
|
|
129
|
+
const sourceErrors = issues.filter((issue) => issue.kind === 'source_validation').map((issue) => issue.message);
|
|
130
|
+
if (sourceErrors.length > 0) {
|
|
131
|
+
findings.push({ kind: 'invalid_source', connectionId, sourceName, errors: sourceErrors });
|
|
132
|
+
}
|
|
133
|
+
for (const issue of issues) {
|
|
134
|
+
if (issue.kind === 'missing_join_target') {
|
|
135
|
+
findings.push({
|
|
136
|
+
kind: 'missing_join_target',
|
|
137
|
+
ownerConnectionId: connectionId,
|
|
138
|
+
ownerSourceName: sourceName,
|
|
139
|
+
targetSourceName: issue.targetSourceName,
|
|
140
|
+
message: issue.message,
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
}
|
|
80
144
|
}
|
|
145
|
+
findings.push(...(await validateWikiSlRefs(input)));
|
|
146
|
+
findings.push(...(await validateWikiRefs(input)));
|
|
81
147
|
for (const pageKey of input.changedWikiPageKeys) {
|
|
82
148
|
const page = await input.wikiService.readPage('GLOBAL', null, pageKey);
|
|
83
149
|
if (!page) {
|
|
84
150
|
continue;
|
|
85
151
|
}
|
|
86
|
-
|
|
152
|
+
findings.push(...(await findInvalidWikiBodyRefIssues({
|
|
87
153
|
pageKey,
|
|
88
154
|
body: page.content,
|
|
89
155
|
visibleConnectionIds: input.connectionIds,
|
|
@@ -94,13 +160,14 @@ export async function validateFinalIngestArtifacts(input) {
|
|
|
94
160
|
tableExists: input.tableExists,
|
|
95
161
|
})));
|
|
96
162
|
}
|
|
97
|
-
|
|
98
|
-
throw new Error(`final artifact gates failed:\n${errors.join('\n')}`);
|
|
99
|
-
}
|
|
163
|
+
return { ok: findings.length === 0, findings };
|
|
100
164
|
}
|
|
101
165
|
export function validateProvenanceRawPaths(input) {
|
|
166
|
+
const currentRawPaths = new Set([...input.currentRawPaths].map(normalizeRawPath));
|
|
167
|
+
const deletedRawPaths = new Set([...input.deletedRawPaths].map(normalizeRawPath));
|
|
102
168
|
for (const row of input.rows) {
|
|
103
|
-
|
|
169
|
+
const rawPath = normalizeRawPath(row.rawPath);
|
|
170
|
+
if (!currentRawPaths.has(rawPath) && !deletedRawPaths.has(rawPath)) {
|
|
104
171
|
throw new Error(`provenance row references raw path outside this snapshot: ${row.rawPath}`);
|
|
105
172
|
}
|
|
106
173
|
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import type { KtxFileStorePort } from '../core/file-store.js';
|
|
2
|
+
import type { KnowledgeWikiService } from '../wiki/knowledge-wiki.service.js';
|
|
3
|
+
import type { FinalArtifactGateFinding } from './artifact-gates.js';
|
|
4
|
+
import type { IngestTraceWriter } from './ingest-trace.js';
|
|
5
|
+
type FinalGatePrunedReferenceKind = 'join' | 'wiki_ref' | 'wiki_sl_ref' | 'wiki_body_ref';
|
|
6
|
+
type SemanticLayerFileStore = Pick<KtxFileStorePort, 'readFile' | 'writeFile' | 'deleteFile' | 'listFiles'>;
|
|
7
|
+
export interface FinalGatePrunedReference {
|
|
8
|
+
kind: FinalGatePrunedReferenceKind;
|
|
9
|
+
artifact: string;
|
|
10
|
+
removedRef: string;
|
|
11
|
+
absentTarget: string;
|
|
12
|
+
}
|
|
13
|
+
export interface FinalGateDroppedSource {
|
|
14
|
+
connectionId: string;
|
|
15
|
+
sourceName: string;
|
|
16
|
+
reason: string;
|
|
17
|
+
}
|
|
18
|
+
export interface FinalGatePruneResult {
|
|
19
|
+
prunedReferences: FinalGatePrunedReference[];
|
|
20
|
+
droppedSources: FinalGateDroppedSource[];
|
|
21
|
+
}
|
|
22
|
+
interface PruneInput {
|
|
23
|
+
workdir: string;
|
|
24
|
+
semanticLayerFiles: SemanticLayerFileStore;
|
|
25
|
+
findings: FinalArtifactGateFinding[];
|
|
26
|
+
droppedSources: FinalGateDroppedSource[];
|
|
27
|
+
trace: IngestTraceWriter;
|
|
28
|
+
author: {
|
|
29
|
+
name: string;
|
|
30
|
+
email: string;
|
|
31
|
+
};
|
|
32
|
+
wikiService?: KnowledgeWikiService;
|
|
33
|
+
}
|
|
34
|
+
export declare function pruneFinalGateFindings(input: PruneInput): Promise<FinalGatePruneResult>;
|
|
35
|
+
export {};
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
import YAML from 'yaml';
|
|
2
|
+
import { listSlSourceFiles, resolveSlSourceFile, slSourceNameForFile } from '../sl/source-files.js';
|
|
3
|
+
async function resolveYamlSource(fileStore, connectionId, sourceName) {
|
|
4
|
+
const file = await resolveSlSourceFile(fileStore, connectionId, sourceName);
|
|
5
|
+
if (!file) {
|
|
6
|
+
return null;
|
|
7
|
+
}
|
|
8
|
+
const parsed = YAML.parse(file.content);
|
|
9
|
+
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
|
10
|
+
throw new Error(`${file.path}: expected semantic-layer source YAML object`);
|
|
11
|
+
}
|
|
12
|
+
return { path: file.path, source: parsed };
|
|
13
|
+
}
|
|
14
|
+
async function writeYamlSource(input) {
|
|
15
|
+
await input.fileStore.writeFile(input.path, YAML.stringify(input.source, { indent: 2, lineWidth: 0, version: '1.1' }), input.author.name, input.author.email, `Prune dangling joins from ${input.path}`, { skipLock: true });
|
|
16
|
+
}
|
|
17
|
+
function removeInlineToken(content, rawToken) {
|
|
18
|
+
return content.replaceAll(`\`${rawToken}\``, '').replace(/[ \t]+([.,;:!?])/g, '$1');
|
|
19
|
+
}
|
|
20
|
+
function escapeRegExp(value) {
|
|
21
|
+
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
22
|
+
}
|
|
23
|
+
function removeWikiRefToken(content, targetPageKey) {
|
|
24
|
+
const pattern = new RegExp(`\\[\\[\\s*${escapeRegExp(targetPageKey)}(?:\\|[^\\]\\n]+)?\\s*\\]\\]`, 'g');
|
|
25
|
+
return content.replace(pattern, '').replace(/[ \t]+([.,;:!?])/g, '$1');
|
|
26
|
+
}
|
|
27
|
+
function wikiBodyAbsentTarget(finding) {
|
|
28
|
+
if (finding.kind === 'missing_wiki_body_table') {
|
|
29
|
+
return finding.tableRef;
|
|
30
|
+
}
|
|
31
|
+
if (finding.kind === 'missing_wiki_body_sl_source') {
|
|
32
|
+
return finding.sourceName;
|
|
33
|
+
}
|
|
34
|
+
if (finding.kind === 'missing_wiki_body_sl_entity') {
|
|
35
|
+
return `${finding.sourceName}.${finding.entityName}`;
|
|
36
|
+
}
|
|
37
|
+
return '';
|
|
38
|
+
}
|
|
39
|
+
/** Remove every join whose target matches `shouldRemove`, write the file back, and
|
|
40
|
+
* emit one pruned-reference record per distinct removed target. */
|
|
41
|
+
async function pruneJoinsFromSource(input) {
|
|
42
|
+
if (!Array.isArray(input.resolved.source.joins)) {
|
|
43
|
+
return [];
|
|
44
|
+
}
|
|
45
|
+
const removed = new Set();
|
|
46
|
+
const nextJoins = input.resolved.source.joins.filter((entry) => {
|
|
47
|
+
const to = entry && typeof entry === 'object' && 'to' in entry ? entry.to : undefined;
|
|
48
|
+
if (typeof to === 'string' && input.shouldRemove(to)) {
|
|
49
|
+
removed.add(to);
|
|
50
|
+
return false;
|
|
51
|
+
}
|
|
52
|
+
return true;
|
|
53
|
+
});
|
|
54
|
+
if (removed.size === 0) {
|
|
55
|
+
return [];
|
|
56
|
+
}
|
|
57
|
+
input.resolved.source.joins = nextJoins;
|
|
58
|
+
await writeYamlSource({
|
|
59
|
+
fileStore: input.fileStore,
|
|
60
|
+
path: input.resolved.path,
|
|
61
|
+
source: input.resolved.source,
|
|
62
|
+
author: input.author,
|
|
63
|
+
});
|
|
64
|
+
const records = [];
|
|
65
|
+
for (const target of removed) {
|
|
66
|
+
const record = {
|
|
67
|
+
kind: 'join',
|
|
68
|
+
artifact: `semantic-layer/${input.connectionId}/${input.ownerSourceName}`,
|
|
69
|
+
removedRef: target,
|
|
70
|
+
absentTarget: target,
|
|
71
|
+
};
|
|
72
|
+
records.push(record);
|
|
73
|
+
await input.trace.event('info', 'final_gates', 'final_gate_reference_pruned', record);
|
|
74
|
+
}
|
|
75
|
+
return records;
|
|
76
|
+
}
|
|
77
|
+
export async function pruneFinalGateFindings(input) {
|
|
78
|
+
const droppedSources = [...input.droppedSources];
|
|
79
|
+
const prunedReferences = [];
|
|
80
|
+
const droppedKey = new Set(droppedSources.map((source) => `${source.connectionId}:${source.sourceName}`));
|
|
81
|
+
for (const finding of input.findings) {
|
|
82
|
+
if (finding.kind !== 'invalid_source') {
|
|
83
|
+
continue;
|
|
84
|
+
}
|
|
85
|
+
const key = `${finding.connectionId}:${finding.sourceName}`;
|
|
86
|
+
if (droppedKey.has(key)) {
|
|
87
|
+
continue;
|
|
88
|
+
}
|
|
89
|
+
const file = await resolveSlSourceFile(input.semanticLayerFiles, finding.connectionId, finding.sourceName);
|
|
90
|
+
if (!file) {
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
const deleted = await input.semanticLayerFiles.deleteFile(file.path, input.author.name, input.author.email, `Drop invalid source ${finding.connectionId}:${finding.sourceName}`, { skipLock: true });
|
|
94
|
+
if (!deleted) {
|
|
95
|
+
continue;
|
|
96
|
+
}
|
|
97
|
+
const dropped = {
|
|
98
|
+
connectionId: finding.connectionId,
|
|
99
|
+
sourceName: finding.sourceName,
|
|
100
|
+
reason: finding.errors.join('; '),
|
|
101
|
+
};
|
|
102
|
+
droppedSources.push(dropped);
|
|
103
|
+
droppedKey.add(key);
|
|
104
|
+
await input.trace.event('info', 'final_gates', 'final_gate_source_dropped', dropped);
|
|
105
|
+
}
|
|
106
|
+
// A dropped node can leave a join dangling on any owner — including sources
|
|
107
|
+
// untouched by this run, which the touched-scoped gate (and the confirm gate
|
|
108
|
+
// after it) never revisit. Prune those edges directly (D5), or the committed
|
|
109
|
+
// orphan join breaks every SL query on the connection.
|
|
110
|
+
const droppedByConnection = new Map();
|
|
111
|
+
for (const dropped of droppedSources) {
|
|
112
|
+
const names = droppedByConnection.get(dropped.connectionId) ?? new Set();
|
|
113
|
+
names.add(dropped.sourceName);
|
|
114
|
+
droppedByConnection.set(dropped.connectionId, names);
|
|
115
|
+
}
|
|
116
|
+
for (const [connectionId, droppedNames] of droppedByConnection) {
|
|
117
|
+
for (const file of await listSlSourceFiles(input.semanticLayerFiles, connectionId)) {
|
|
118
|
+
let parsed;
|
|
119
|
+
try {
|
|
120
|
+
parsed = YAML.parse(file.content);
|
|
121
|
+
}
|
|
122
|
+
catch {
|
|
123
|
+
continue;
|
|
124
|
+
}
|
|
125
|
+
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
|
126
|
+
continue;
|
|
127
|
+
}
|
|
128
|
+
prunedReferences.push(...(await pruneJoinsFromSource({
|
|
129
|
+
fileStore: input.semanticLayerFiles,
|
|
130
|
+
connectionId,
|
|
131
|
+
ownerSourceName: slSourceNameForFile(file.path, file.content),
|
|
132
|
+
resolved: { path: file.path, source: parsed },
|
|
133
|
+
shouldRemove: (target) => droppedNames.has(target),
|
|
134
|
+
author: input.author,
|
|
135
|
+
trace: input.trace,
|
|
136
|
+
})));
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
for (const finding of input.findings) {
|
|
140
|
+
if (finding.kind !== 'missing_join_target') {
|
|
141
|
+
continue;
|
|
142
|
+
}
|
|
143
|
+
const resolved = await resolveYamlSource(input.semanticLayerFiles, finding.ownerConnectionId, finding.ownerSourceName);
|
|
144
|
+
if (!resolved) {
|
|
145
|
+
continue;
|
|
146
|
+
}
|
|
147
|
+
prunedReferences.push(...(await pruneJoinsFromSource({
|
|
148
|
+
fileStore: input.semanticLayerFiles,
|
|
149
|
+
connectionId: finding.ownerConnectionId,
|
|
150
|
+
ownerSourceName: finding.ownerSourceName,
|
|
151
|
+
resolved,
|
|
152
|
+
shouldRemove: (target) => target === finding.targetSourceName,
|
|
153
|
+
author: input.author,
|
|
154
|
+
trace: input.trace,
|
|
155
|
+
})));
|
|
156
|
+
}
|
|
157
|
+
const wikiFindings = input.findings.filter((finding) => finding.kind === 'missing_wiki_ref' ||
|
|
158
|
+
finding.kind === 'missing_wiki_sl_ref' ||
|
|
159
|
+
finding.kind === 'missing_wiki_body_sl_source' ||
|
|
160
|
+
finding.kind === 'missing_wiki_body_sl_entity' ||
|
|
161
|
+
finding.kind === 'missing_wiki_body_table');
|
|
162
|
+
const pageKeys = [...new Set(wikiFindings.map((finding) => finding.pageKey))].sort();
|
|
163
|
+
for (const pageKey of pageKeys) {
|
|
164
|
+
const page = input.wikiService ? await input.wikiService.readPage('GLOBAL', null, pageKey) : null;
|
|
165
|
+
if (!page) {
|
|
166
|
+
continue;
|
|
167
|
+
}
|
|
168
|
+
const frontmatter = { ...page.frontmatter };
|
|
169
|
+
let content = page.content;
|
|
170
|
+
let changed = false;
|
|
171
|
+
for (const finding of wikiFindings.filter((candidate) => candidate.pageKey === pageKey)) {
|
|
172
|
+
if (finding.kind === 'missing_wiki_ref') {
|
|
173
|
+
const refs = Array.isArray(frontmatter.refs) ? frontmatter.refs.filter((ref) => ref !== finding.targetPageKey) : [];
|
|
174
|
+
const nextContent = removeWikiRefToken(content, finding.targetPageKey);
|
|
175
|
+
if ((Array.isArray(frontmatter.refs) && refs.length !== frontmatter.refs.length) || nextContent !== content) {
|
|
176
|
+
if (Array.isArray(frontmatter.refs)) {
|
|
177
|
+
frontmatter.refs = refs;
|
|
178
|
+
}
|
|
179
|
+
content = nextContent;
|
|
180
|
+
changed = true;
|
|
181
|
+
const record = {
|
|
182
|
+
kind: 'wiki_ref',
|
|
183
|
+
artifact: `wiki/global/${pageKey}`,
|
|
184
|
+
removedRef: finding.targetPageKey,
|
|
185
|
+
absentTarget: finding.targetPageKey,
|
|
186
|
+
};
|
|
187
|
+
prunedReferences.push(record);
|
|
188
|
+
await input.trace.event('info', 'final_gates', 'final_gate_reference_pruned', record);
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
else if (finding.kind === 'missing_wiki_sl_ref') {
|
|
192
|
+
const slRefs = Array.isArray(frontmatter.sl_refs)
|
|
193
|
+
? frontmatter.sl_refs.filter((ref) => ref !== finding.ref)
|
|
194
|
+
: [];
|
|
195
|
+
if (Array.isArray(frontmatter.sl_refs) && slRefs.length !== frontmatter.sl_refs.length) {
|
|
196
|
+
frontmatter.sl_refs = slRefs;
|
|
197
|
+
changed = true;
|
|
198
|
+
const record = {
|
|
199
|
+
kind: 'wiki_sl_ref',
|
|
200
|
+
artifact: `wiki/global/${pageKey}`,
|
|
201
|
+
removedRef: finding.ref,
|
|
202
|
+
absentTarget: finding.sourceName,
|
|
203
|
+
};
|
|
204
|
+
prunedReferences.push(record);
|
|
205
|
+
await input.trace.event('info', 'final_gates', 'final_gate_reference_pruned', record);
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
else {
|
|
209
|
+
const nextContent = removeInlineToken(content, finding.rawToken);
|
|
210
|
+
if (nextContent !== content) {
|
|
211
|
+
content = nextContent;
|
|
212
|
+
changed = true;
|
|
213
|
+
const record = {
|
|
214
|
+
kind: 'wiki_body_ref',
|
|
215
|
+
artifact: `wiki/global/${pageKey}`,
|
|
216
|
+
removedRef: finding.rawToken,
|
|
217
|
+
absentTarget: wikiBodyAbsentTarget(finding),
|
|
218
|
+
};
|
|
219
|
+
prunedReferences.push(record);
|
|
220
|
+
await input.trace.event('info', 'final_gates', 'final_gate_reference_pruned', record);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
if (changed && input.wikiService) {
|
|
225
|
+
await input.wikiService.writePage('GLOBAL', null, pageKey, frontmatter, content, input.author.name, input.author.email, `Prune dangling refs from ${pageKey}`, { skipLock: true });
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
return { prunedReferences, droppedSources };
|
|
229
|
+
}
|
|
@@ -8,6 +8,8 @@ export declare class IngestBundleRunner {
|
|
|
8
8
|
private readonly logger;
|
|
9
9
|
private readonly chainByConnection;
|
|
10
10
|
constructor(deps: IngestBundleRunnerDeps);
|
|
11
|
+
private cachedWorkUnitOutcome;
|
|
12
|
+
private saveSuccessfulWorkUnitCache;
|
|
11
13
|
run(job: IngestBundleJob, ctx?: IngestJobContext): Promise<IngestBundleResult>;
|
|
12
14
|
private formatRateLimitWait;
|
|
13
15
|
private subscribeRateLimitGovernor;
|
|
@@ -61,6 +63,7 @@ export declare class IngestBundleRunner {
|
|
|
61
63
|
private uniqueWikiPageKeys;
|
|
62
64
|
private uniqueTouchedSlSources;
|
|
63
65
|
private removedWikiPageKeysFromActions;
|
|
66
|
+
private markFinalGateDroppedSourceWorkUnits;
|
|
64
67
|
private finalGateActionOrigins;
|
|
65
68
|
private wikiPageKeysForFinalGates;
|
|
66
69
|
private runWorkUnitInWorktree;
|