@weftlet/toolkit 0.1.2 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -3
- package/THIRD_PARTY_NOTICES.md +33 -0
- package/dist/application/hydrator.js +21 -6
- package/dist/application/overlay.js +46 -9
- package/dist/application/saved-evidence.d.ts +1 -0
- package/dist/configuration/config.d.ts +4 -0
- package/dist/configuration/config.js +12 -0
- package/dist/configuration/scip-project.d.ts +9 -0
- package/dist/configuration/scip-project.js +40 -0
- package/dist/extraction/codex-auth.js +9 -2
- package/dist/extraction/codex-command.d.ts +8 -0
- package/dist/extraction/codex-command.js +81 -0
- package/dist/extraction/document-references.d.ts +18 -0
- package/dist/extraction/document-references.js +94 -0
- package/dist/extraction/document.d.ts +9 -2
- package/dist/extraction/document.js +40 -47
- package/dist/extraction/module-request.d.ts +3 -0
- package/dist/extraction/module-request.js +25 -0
- package/dist/extraction/repository-nodes.d.ts +7 -0
- package/dist/extraction/repository-nodes.js +20 -0
- package/dist/extraction/routing.d.ts +14 -0
- package/dist/extraction/routing.js +151 -0
- package/dist/extraction/scip-project.d.ts +3 -0
- package/dist/extraction/scip-project.js +33 -0
- package/dist/extraction/semantic-process.js +17 -5
- package/dist/extraction/semantic.js +32 -5
- package/dist/extraction/static.d.ts +7 -3
- package/dist/extraction/static.js +15 -9
- package/dist/graph/query.js +21 -34
- package/dist/graph/ranking.d.ts +10 -0
- package/dist/graph/ranking.js +85 -0
- package/dist/interfaces/doctor.js +12 -3
- package/docs/ADR-002-saved-evidence.md +7 -1
- package/docs/ADR-003-draft-overlays.md +6 -0
- package/docs/ADR-004-rdf-query-path.md +6 -0
- package/docs/ADR-006-provider-boundaries.md +5 -0
- package/docs/ADR-010-committed-source-indexing.md +77 -0
- package/docs/ADR-011-explicit-document-links.md +48 -0
- package/docs/ADR-012-impact-ranking.md +50 -0
- package/docs/ADR-013-relative-module-requests.md +58 -0
- package/docs/ADR-014-routing-declarations.md +108 -0
- package/docs/ADR-015-repository-document-identity.md +57 -0
- package/docs/CONSUMER-TRIAL.md +166 -0
- package/docs/ENVIRONMENT-VALIDATION.md +9 -0
- package/docs/IMPACT-EVALUATION.md +363 -0
- package/docs/PROVIDERS.md +30 -2
- package/docs/RELEASE-1.0-PLAN.md +221 -0
- package/docs/RELEASING.md +16 -6
- package/docs/ROADMAP.md +53 -5
- package/docs/SOURCE-INVENTORY.md +10 -0
- package/docs/VALIDATION.md +438 -0
- package/docs/VIEWER.md +5 -0
- package/docs/WEFTLET-2.0-PLAN.md +282 -0
- package/docs/extraction-manifest.json +94 -17
- package/package.json +5 -2
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import { posix } from 'node:path';
|
|
2
|
+
import { fromMarkdown } from 'mdast-util-from-markdown';
|
|
3
|
+
// Markdown destinations are document-relative. Literal paths are repository-relative unless
|
|
4
|
+
// prefixed with ./ or ../. Parsing Markdown keeps examples, URLs and link labels out of the graph.
|
|
5
|
+
const barePath = /(?<![\w/:@.-])(?:\.{1,2}\/|\/)?[A-Za-z0-9_.-]+(?:\/[A-Za-z0-9_.-]+)+/g;
|
|
6
|
+
const fileName = /\.(?:md|mdx|txt|ts|tsx|mts|cts|js|jsx|mjs|cjs|json|ya?ml|prisma|sh)$/i;
|
|
7
|
+
export function documentReferences(content) {
|
|
8
|
+
const tree = fromMarkdown(content);
|
|
9
|
+
const definitions = new Map();
|
|
10
|
+
const walk = (node, visit) => {
|
|
11
|
+
visit(node);
|
|
12
|
+
if ('children' in node)
|
|
13
|
+
for (const child of node.children)
|
|
14
|
+
walk(child, visit);
|
|
15
|
+
};
|
|
16
|
+
walk(tree, (node) => {
|
|
17
|
+
if (node.type === 'definition' && !definitions.has(node.identifier)) {
|
|
18
|
+
definitions.set(node.identifier, node);
|
|
19
|
+
}
|
|
20
|
+
});
|
|
21
|
+
const references = [];
|
|
22
|
+
const add = (node, target, relativeTo) => {
|
|
23
|
+
if (node.position)
|
|
24
|
+
references.push({ target, relativeTo,
|
|
25
|
+
startLine: node.position.start.line, endLine: node.position.end.line });
|
|
26
|
+
};
|
|
27
|
+
const visit = (node) => {
|
|
28
|
+
if (node.type === 'link') {
|
|
29
|
+
add(node, node.url, 'document');
|
|
30
|
+
return;
|
|
31
|
+
}
|
|
32
|
+
if (node.type === 'linkReference') {
|
|
33
|
+
const definition = definitions.get(node.identifier);
|
|
34
|
+
// Cite the declaration containing the destination, not a path invented from its label.
|
|
35
|
+
if (definition)
|
|
36
|
+
add(definition, definition.url, 'document');
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
if (node.type === 'inlineCode') {
|
|
40
|
+
if (!/\s/.test(node.value) && (node.value.includes('/') || fileName.test(node.value))) {
|
|
41
|
+
add(node, node.value, 'repository');
|
|
42
|
+
}
|
|
43
|
+
return;
|
|
44
|
+
}
|
|
45
|
+
if (node.type === 'text' && node.position) {
|
|
46
|
+
const raw = content.slice(node.position.start.offset, node.position.end.offset);
|
|
47
|
+
for (const match of raw.matchAll(barePath)) {
|
|
48
|
+
const target = match[0].replace(/[.,;:]+$/, '');
|
|
49
|
+
if (!/[A-Za-z]/.test(target))
|
|
50
|
+
continue;
|
|
51
|
+
const startLine = node.position.start.line + raw.slice(0, match.index).split('\n').length - 1;
|
|
52
|
+
references.push({ target, startLine, endLine: startLine, relativeTo: 'repository' });
|
|
53
|
+
}
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
// Code blocks, images, HTML and unused reference definitions are not repository mappings.
|
|
57
|
+
if ('children' in node)
|
|
58
|
+
for (const child of node.children)
|
|
59
|
+
visit(child);
|
|
60
|
+
};
|
|
61
|
+
visit(tree);
|
|
62
|
+
return references;
|
|
63
|
+
}
|
|
64
|
+
export function resolveDocumentReference(sourcePath, reference, tracked) {
|
|
65
|
+
const target = reference.target;
|
|
66
|
+
// No network, filesystem lookup or guessed basename fallback. A leading / denotes repository root.
|
|
67
|
+
if (!target || target.startsWith('#') || target.startsWith('//') || /^[a-z][\w+.-]*:/i.test(target)) {
|
|
68
|
+
return { kind: 'ignored' };
|
|
69
|
+
}
|
|
70
|
+
let path;
|
|
71
|
+
try {
|
|
72
|
+
path = decodeURIComponent(target.split(/[?#]/, 1)[0]);
|
|
73
|
+
}
|
|
74
|
+
catch {
|
|
75
|
+
return { kind: 'missing', reason: `Malformed repository link destination: ${target}` };
|
|
76
|
+
}
|
|
77
|
+
if (/[\\\x00-\x1f\x7f]/.test(path)) {
|
|
78
|
+
return { kind: 'missing', reason: `Unsupported repository link destination: ${target}` };
|
|
79
|
+
}
|
|
80
|
+
const relative = reference.relativeTo === 'document' || /^\.{1,2}\//.test(path);
|
|
81
|
+
path = posix.normalize(path.startsWith('/') ? path.slice(1)
|
|
82
|
+
: posix.join(relative ? posix.dirname(sourcePath) : '', path));
|
|
83
|
+
if (path === '..' || path.startsWith('../') || posix.isAbsolute(path)) {
|
|
84
|
+
return { kind: 'missing', reason: `Repository link escapes the repository: ${target}` };
|
|
85
|
+
}
|
|
86
|
+
if (tracked.has(path))
|
|
87
|
+
return { kind: 'resolved', path };
|
|
88
|
+
// Slash-separated prose (RDF/SHACL), package names and example paths are not declarations
|
|
89
|
+
// of missing repository files. Only an explicit Markdown destination requests link validation.
|
|
90
|
+
if (reference.relativeTo === 'repository')
|
|
91
|
+
return { kind: 'ignored' };
|
|
92
|
+
return { kind: 'missing', path,
|
|
93
|
+
reason: `Document reference ${target} resolves to ${path}, which is not a committed file. Directory references are not expanded.` };
|
|
94
|
+
}
|
|
@@ -1,5 +1,12 @@
|
|
|
1
|
-
import type { ExtractionContext, GraphFact, SemanticArtifact, SemanticSource } from
|
|
1
|
+
import type { ExtractionContext, GraphDiagnostic, GraphFact, SemanticArtifact, SemanticSource } from '@weftlet/core';
|
|
2
|
+
export declare const documentLinkerIdentity: {
|
|
3
|
+
readonly id: "document-linker";
|
|
4
|
+
readonly version: "2.0.0";
|
|
5
|
+
};
|
|
2
6
|
export declare function linkSemanticDocuments(context: ExtractionContext, hydrated: readonly {
|
|
3
7
|
source: SemanticSource;
|
|
4
8
|
artifact: SemanticArtifact;
|
|
5
|
-
}[], trackedPaths
|
|
9
|
+
}[], trackedPaths: readonly string[], approvedSourcePaths?: readonly string[]): {
|
|
10
|
+
readonly facts: readonly GraphFact[];
|
|
11
|
+
readonly diagnostics: readonly GraphDiagnostic[];
|
|
12
|
+
};
|
|
@@ -1,54 +1,47 @@
|
|
|
1
|
-
import { sha256, stableJson } from
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
const
|
|
9
|
-
const makeFact = (source, subject, predicate, object) => ({
|
|
10
|
-
id: sha256(stableJson({ source: source.path, subject: subject.id, predicate, object: object.id })),
|
|
11
|
-
subject,
|
|
12
|
-
predicate,
|
|
13
|
-
object,
|
|
14
|
-
provenance: {
|
|
15
|
-
path: source.path,
|
|
16
|
-
startLine: 1,
|
|
17
|
-
endLine: Math.max(1, source.content.split('\n').length),
|
|
18
|
-
digest: source.digest,
|
|
19
|
-
commitSha: context.commitSha,
|
|
20
|
-
},
|
|
21
|
-
extractor: identity,
|
|
22
|
-
confidence: 1,
|
|
23
|
-
authority: 'inferred',
|
|
24
|
-
extractionReview: predicate === 'contains' ? 'unreviewed' : 'deterministic',
|
|
25
|
-
snapshotId: context.snapshotId,
|
|
26
|
-
});
|
|
1
|
+
import { sha256, stableJson } from '../infrastructure/util.js';
|
|
2
|
+
import { documentReferences, resolveDocumentReference } from './document-references.js';
|
|
3
|
+
import { repositoryFileNode as fileNode } from './repository-nodes.js';
|
|
4
|
+
export const documentLinkerIdentity = { id: 'document-linker', version: '2.0.0' };
|
|
5
|
+
export function linkSemanticDocuments(context, hydrated, trackedPaths, approvedSourcePaths = []) {
|
|
6
|
+
const facts = new Map();
|
|
7
|
+
const diagnostics = new Map();
|
|
8
|
+
const tracked = new Set(trackedPaths);
|
|
27
9
|
for (const { source, artifact } of hydrated) {
|
|
28
|
-
const owner =
|
|
29
|
-
const
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
const
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
10
|
+
const owner = { id: `document:${source.path}`, kind: 'Document', label: source.path };
|
|
11
|
+
const span = (startLine, endLine) => ({
|
|
12
|
+
path: source.path, startLine, endLine, digest: source.digest, commitSha: context.commitSha,
|
|
13
|
+
});
|
|
14
|
+
const add = (predicate, object, provenance) => {
|
|
15
|
+
const id = sha256(stableJson({ subject: owner.id, predicate, object: object.id, provenance,
|
|
16
|
+
extractor: documentLinkerIdentity }));
|
|
17
|
+
facts.set(id, { id, subject: owner, predicate, object, provenance,
|
|
18
|
+
extractor: documentLinkerIdentity, confidence: 1, authority: 'inferred',
|
|
19
|
+
sourceAuthority: approvedSourcePaths.includes(source.path) ? 'declared' : 'inferred',
|
|
20
|
+
extractionReview: predicate === 'contains' ? 'unreviewed' : 'deterministic',
|
|
21
|
+
snapshotId: context.snapshotId });
|
|
22
|
+
};
|
|
23
|
+
// Containment follows the assertion's actual cited passage and stays unreviewed.
|
|
24
|
+
for (const fact of artifact.facts) {
|
|
25
|
+
for (const node of [fact.subject, fact.object]) {
|
|
26
|
+
add('contains', node, span(fact.provenance.startLine, fact.provenance.endLine));
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
for (const reference of documentReferences(source.content)) {
|
|
30
|
+
const resolved = resolveDocumentReference(source.path, reference, tracked);
|
|
31
|
+
if (resolved.kind === 'ignored')
|
|
39
32
|
continue;
|
|
33
|
+
const provenance = span(reference.startLine, reference.endLine);
|
|
34
|
+
if (resolved.kind === 'resolved') {
|
|
35
|
+
add('references', fileNode(resolved.path), provenance);
|
|
40
36
|
}
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
37
|
+
else {
|
|
38
|
+
const id = sha256(stableJson({ provenance, reason: resolved.reason, extractor: documentLinkerIdentity }));
|
|
39
|
+
diagnostics.set(id, { id, kind: 'implementationGaps', reason: resolved.reason,
|
|
40
|
+
provenance: [provenance], relatedNodeIds: [owner.id, ...(resolved.path ? [fileNode(resolved.path).id] : [])],
|
|
41
|
+
evidenceScope: 'source' });
|
|
44
42
|
}
|
|
45
43
|
}
|
|
46
|
-
for (const path of [...resolvedReferences].sort()) {
|
|
47
|
-
const target = /\.(?:md|mdx|txt)$/i.test(path)
|
|
48
|
-
? documentNode(path)
|
|
49
|
-
: { id: `component:file:${path}`, kind: 'Component', label: path };
|
|
50
|
-
facts.push(makeFact(source, owner, 'references', target));
|
|
51
|
-
}
|
|
52
44
|
}
|
|
53
|
-
return facts
|
|
45
|
+
return { facts: [...facts.values()].sort((a, b) => a.id.localeCompare(b.id)),
|
|
46
|
+
diagnostics: [...diagnostics.values()].sort((a, b) => a.id.localeCompare(b.id)) };
|
|
54
47
|
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { posix } from 'node:path';
|
|
2
|
+
import { sha256, stableJson } from '../infrastructure/util.js';
|
|
3
|
+
/** Identity of a literal module request, not a resolved implementation or an existence check. */
|
|
4
|
+
export function moduleRequestNode(importer, specifier) {
|
|
5
|
+
const relative = specifier === '.' || specifier === '..' ||
|
|
6
|
+
specifier.startsWith('./') || specifier.startsWith('../');
|
|
7
|
+
if (!relative)
|
|
8
|
+
return { id: `contract:module:${specifier}`, kind: 'Contract', label: specifier };
|
|
9
|
+
// Git paths use POSIX separators on every host. Do not interpret loader suffixes, escapes,
|
|
10
|
+
// encoded paths or requests outside this repository as canonical local destinations.
|
|
11
|
+
const location = posix.normalize(posix.join(posix.dirname(importer), specifier));
|
|
12
|
+
const opaque = /[\\%?#\x00-\x1f\x7f-\x9f]/.test(specifier) || importer.includes('\\') ||
|
|
13
|
+
location === '..' || location.startsWith('../') || posix.isAbsolute(location);
|
|
14
|
+
if (opaque)
|
|
15
|
+
return {
|
|
16
|
+
id: `contract:module:./?${sha256(stableJson({ importer, specifier }))}`,
|
|
17
|
+
kind: 'Contract', label: `${specifier} (from ${importer})`,
|
|
18
|
+
attributes: { identityScope: 'importer-local-request', resolution: 'unresolved' },
|
|
19
|
+
};
|
|
20
|
+
// The ./ prefix cannot collide with a bare import; ? is reserved for opaque identities.
|
|
21
|
+
// Encode the location so distinct literal path characters cannot act as ID delimiters.
|
|
22
|
+
return { id: `contract:module:./${encodeURIComponent(location)}`, kind: 'Contract',
|
|
23
|
+
label: `./${location}`,
|
|
24
|
+
attributes: { identityScope: 'repository-relative-request', resolution: 'unresolved' } };
|
|
25
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { GraphNode } from '@weftlet/core';
|
|
2
|
+
/** Internal cache contract: changing this policy must also change snapshot assembly identity. */
|
|
3
|
+
export declare const repositoryNodeIdentityVersion = 1;
|
|
4
|
+
/** Physical file identity; this does not establish existence, intent, ownership or approval. */
|
|
5
|
+
export declare function repositoryFileNode(path: string): GraphNode;
|
|
6
|
+
/** Normalize only reserved document/file identities, never inferred business concept IDs. */
|
|
7
|
+
export declare function canonicalizeRepositoryDocumentNode(node: GraphNode): GraphNode;
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/** Internal cache contract: changing this policy must also change snapshot assembly identity. */
|
|
2
|
+
export const repositoryNodeIdentityVersion = 1;
|
|
3
|
+
function isDocumentPath(path) {
|
|
4
|
+
return /\.(?:md|mdx|txt)$/i.test(path);
|
|
5
|
+
}
|
|
6
|
+
/** Physical file identity; this does not establish existence, intent, ownership or approval. */
|
|
7
|
+
export function repositoryFileNode(path) {
|
|
8
|
+
if (isDocumentPath(path))
|
|
9
|
+
return { id: `document:${path}`, kind: 'Document', label: path };
|
|
10
|
+
return { id: `component:file:${path}`, label: path,
|
|
11
|
+
kind: /\.(spec|test)\.[^.]+$/i.test(path) ? 'Evidence' : 'Component' };
|
|
12
|
+
}
|
|
13
|
+
/** Normalize only reserved document/file identities, never inferred business concept IDs. */
|
|
14
|
+
export function canonicalizeRepositoryDocumentNode(node) {
|
|
15
|
+
const match = /^(?:component:file:|document:)(.+)$/.exec(node.id);
|
|
16
|
+
if (!match || !isDocumentPath(match[1]))
|
|
17
|
+
return node;
|
|
18
|
+
const canonical = repositoryFileNode(match[1]);
|
|
19
|
+
return { ...node, id: canonical.id, kind: canonical.kind };
|
|
20
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { ExtractionContext, GraphDiagnostic, GraphFact } from '@weftlet/core';
|
|
2
|
+
import type { GitRepository } from '../repository/git.js';
|
|
3
|
+
export declare const routingExtractorIdentity: {
|
|
4
|
+
readonly id: "routing-declaration-extractor";
|
|
5
|
+
readonly version: "1.0.0";
|
|
6
|
+
};
|
|
7
|
+
/** Reads explicit mapping declarations, never infers routing from matching topic text. */
|
|
8
|
+
export declare function extractRoutingDeclarations(repository: GitRepository, context: ExtractionContext, paths: readonly string[], options?: {
|
|
9
|
+
readonly allowMissingSources?: boolean;
|
|
10
|
+
}): Promise<{
|
|
11
|
+
readonly facts: readonly GraphFact[];
|
|
12
|
+
readonly diagnostics: readonly GraphDiagnostic[];
|
|
13
|
+
readonly referencedPaths: readonly string[];
|
|
14
|
+
}>;
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
import { posix } from 'node:path';
|
|
2
|
+
import { sha256, stableJson } from '../infrastructure/util.js';
|
|
3
|
+
export const routingExtractorIdentity = { id: 'routing-declaration-extractor', version: '1.0.0' };
|
|
4
|
+
function record(value) {
|
|
5
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
6
|
+
}
|
|
7
|
+
function exactKeys(value, keys) {
|
|
8
|
+
return Object.keys(value).length === keys.length && keys.every((key) => Object.hasOwn(value, key));
|
|
9
|
+
}
|
|
10
|
+
function label(value) {
|
|
11
|
+
return typeof value === 'string' && value.length > 0 && value.length <= 200 &&
|
|
12
|
+
value === value.trim() && !/[\x00-\x1f\x7f-\x9f]/.test(value);
|
|
13
|
+
}
|
|
14
|
+
function repositoryPath(value) {
|
|
15
|
+
return typeof value === 'string' && value.length > 0 && value.length <= 1024 &&
|
|
16
|
+
value === value.trim() && !/[\\:*?\[\]{}%#\x00-\x1f\x7f-\x9f]/.test(value) &&
|
|
17
|
+
!posix.isAbsolute(value) && posix.normalize(value) === value &&
|
|
18
|
+
value.split('/').every((part) => part !== '' && part !== '.' && part !== '..');
|
|
19
|
+
}
|
|
20
|
+
/** JSON.parse alone silently accepts duplicate keys, which would hide conflicting declarations. */
|
|
21
|
+
function parseUniqueJson(content, path) {
|
|
22
|
+
let parsed;
|
|
23
|
+
try {
|
|
24
|
+
parsed = JSON.parse(content);
|
|
25
|
+
}
|
|
26
|
+
catch {
|
|
27
|
+
throw new Error(`Invalid routing source ${path}: malformed JSON`);
|
|
28
|
+
}
|
|
29
|
+
const objects = [];
|
|
30
|
+
for (const token of content.matchAll(/"(?:\\.|[^"\\])*"|[{}\[\]]/g)) {
|
|
31
|
+
const value = token[0];
|
|
32
|
+
if (value === '{' || value === '[')
|
|
33
|
+
objects.push(value === '{' ? new Set() : undefined);
|
|
34
|
+
else if (value === '}' || value === ']')
|
|
35
|
+
objects.pop();
|
|
36
|
+
else if (/^\s*:/.test(content.slice(token.index + value.length))) {
|
|
37
|
+
const keys = objects.at(-1);
|
|
38
|
+
const key = JSON.parse(value);
|
|
39
|
+
if (keys.has(key))
|
|
40
|
+
throw new Error(`Invalid routing source ${path}: duplicate JSON key ${key}`);
|
|
41
|
+
keys.add(key);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
return parsed;
|
|
45
|
+
}
|
|
46
|
+
function parseRoutes(content, path) {
|
|
47
|
+
if (Buffer.byteLength(content) > 65_536)
|
|
48
|
+
throw new Error(`Routing source ${path} exceeds 64 KiB`);
|
|
49
|
+
const parsed = parseUniqueJson(content, path);
|
|
50
|
+
if (!record(parsed) || !exactKeys(parsed, ['schemaVersion', 'routes']) ||
|
|
51
|
+
parsed.schemaVersion !== 1 || !Array.isArray(parsed.routes)) {
|
|
52
|
+
throw new Error(`Invalid routing source ${path}: expected schemaVersion 1 and routes array`);
|
|
53
|
+
}
|
|
54
|
+
return parsed.routes.map((route, index) => {
|
|
55
|
+
if (!record(route) || !exactKeys(route, ['id', 'scope', 'channel', 'producer', 'consumer']) ||
|
|
56
|
+
!label(route.id) || !label(route.scope) || !label(route.channel) ||
|
|
57
|
+
!repositoryPath(route.producer) || !repositoryPath(route.consumer)) {
|
|
58
|
+
throw new Error(`Invalid routing source ${path}: route ${index + 1} requires exact id/scope/channel labels and normalized producer/consumer paths`);
|
|
59
|
+
}
|
|
60
|
+
return { id: route.id, scope: route.scope, channel: route.channel,
|
|
61
|
+
producer: route.producer, consumer: route.consumer };
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
function fileNode(path) {
|
|
65
|
+
return { id: `component:file:${path}`, label: path,
|
|
66
|
+
kind: /\.(spec|test)\.[^.]+$/i.test(path) ? 'Evidence' : 'Component' };
|
|
67
|
+
}
|
|
68
|
+
/** Reads explicit mapping declarations, never infers routing from matching topic text. */
|
|
69
|
+
export async function extractRoutingDeclarations(repository, context, paths, options = {}) {
|
|
70
|
+
if (paths.length > 16 || new Set(paths).size !== paths.length ||
|
|
71
|
+
paths.some((path) => !repositoryPath(path) || !path.endsWith('.json'))) {
|
|
72
|
+
throw new Error('Routing sources require at most 16 unique exact repository-relative .json paths');
|
|
73
|
+
}
|
|
74
|
+
if (paths.length === 0)
|
|
75
|
+
return { facts: [], diagnostics: [], referencedPaths: [] };
|
|
76
|
+
// NUL-delimited tree entries preserve spaces/Unicode regardless of core.quotepath.
|
|
77
|
+
// A symlink or submodule is not evidence of a regular source file at this path.
|
|
78
|
+
const entries = (await repository.run(['ls-tree', '-r', '-z', context.commitSha, '--', '.']))
|
|
79
|
+
.split('\0').filter(Boolean).map((entry) => {
|
|
80
|
+
const fields = /^(\d{6}) (blob|tree|commit) [a-f0-9]+\t([\s\S]+)$/.exec(entry);
|
|
81
|
+
if (!fields)
|
|
82
|
+
throw new Error('Unable to read exact committed routing file entries');
|
|
83
|
+
return { mode: fields[1], type: fields[2], path: fields[3] };
|
|
84
|
+
});
|
|
85
|
+
const tracked = new Set(entries.map((entry) => entry.path));
|
|
86
|
+
const regular = new Set(entries.filter((entry) => entry.type === 'blob' &&
|
|
87
|
+
['100644', '100755'].includes(entry.mode)).map((entry) => entry.path));
|
|
88
|
+
const cited = [];
|
|
89
|
+
const diagnostics = [];
|
|
90
|
+
const addDiagnostic = (reason, provenance, relatedNodeIds) => {
|
|
91
|
+
diagnostics.push({ id: sha256(stableJson({ reason, provenance, relatedNodeIds,
|
|
92
|
+
extractor: routingExtractorIdentity, commitSha: context.commitSha })),
|
|
93
|
+
kind: 'implementationGaps', reason, provenance, relatedNodeIds, evidenceScope: 'source' });
|
|
94
|
+
};
|
|
95
|
+
for (const path of [...paths].sort()) {
|
|
96
|
+
if (!tracked.has(path)) {
|
|
97
|
+
if (!options.allowMissingSources)
|
|
98
|
+
throw new Error(`Routing source ${path} is not a committed file at ${context.commitSha}`);
|
|
99
|
+
addDiagnostic(`Routing source ${path} is absent from this draft; no bindings from that source were retained.`, [], [`document:${path}`]);
|
|
100
|
+
continue;
|
|
101
|
+
}
|
|
102
|
+
if (!regular.has(path))
|
|
103
|
+
throw new Error(`Routing source ${path} is not a regular committed file at ${context.commitSha}`);
|
|
104
|
+
const content = await repository.readFileAt(context.commitSha, path);
|
|
105
|
+
const provenance = { path, startLine: 1, endLine: content.split('\n').length,
|
|
106
|
+
digest: sha256(content), commitSha: context.commitSha };
|
|
107
|
+
cited.push(...parseRoutes(content, path).map((route) => ({ route, provenance })));
|
|
108
|
+
if (cited.length > 500)
|
|
109
|
+
throw new Error('Routing sources exceed the 500 declaration bound');
|
|
110
|
+
}
|
|
111
|
+
const groups = new Map();
|
|
112
|
+
const referencedPaths = new Set();
|
|
113
|
+
for (const item of cited) {
|
|
114
|
+
const key = stableJson({ repositoryId: context.repositoryId, scope: item.route.scope, id: item.route.id });
|
|
115
|
+
const group = groups.get(key) ?? [];
|
|
116
|
+
group.push(item);
|
|
117
|
+
groups.set(key, group);
|
|
118
|
+
referencedPaths.add(item.route.producer);
|
|
119
|
+
referencedPaths.add(item.route.consumer);
|
|
120
|
+
}
|
|
121
|
+
const facts = [];
|
|
122
|
+
for (const [key, group] of groups) {
|
|
123
|
+
const { route, provenance } = group[0];
|
|
124
|
+
const binding = { id: `workflow:routing:${sha256(key)}`, kind: 'Workflow',
|
|
125
|
+
label: `Routing ${route.scope}/${route.id}: ${route.channel}`,
|
|
126
|
+
attributes: { evidenceLayer: 'routing-declaration', routeId: route.id, scope: route.scope,
|
|
127
|
+
channel: route.channel } };
|
|
128
|
+
if (group.length > 1) {
|
|
129
|
+
addDiagnostic(`Ambiguous routing declaration ${route.scope}/${route.id}: ${group.length} records share this scoped identity; all of its bindings were suppressed.`, [...new Map(group.map((item) => [stableJson(item.provenance), item.provenance])).values()], [binding.id, ...new Set(group.flatMap((item) => [fileNode(item.route.producer).id, fileNode(item.route.consumer).id]))]);
|
|
130
|
+
continue;
|
|
131
|
+
}
|
|
132
|
+
const missing = [...new Set([route.producer, route.consumer])].filter((path) => !regular.has(path));
|
|
133
|
+
if (missing.length > 0) {
|
|
134
|
+
addDiagnostic(`Routing declaration ${route.scope}/${route.id} names missing or non-regular committed endpoint(s): ${missing.join(', ')}. Both binding relationships were suppressed; no path or case was guessed.`, [provenance], [binding.id, fileNode(route.producer).id, fileNode(route.consumer).id]);
|
|
135
|
+
continue;
|
|
136
|
+
}
|
|
137
|
+
const relationships = [
|
|
138
|
+
{ subject: fileNode(route.producer), predicate: 'emits', object: binding },
|
|
139
|
+
{ subject: binding, predicate: 'routesThrough', object: fileNode(route.consumer) },
|
|
140
|
+
];
|
|
141
|
+
for (const relationship of relationships) {
|
|
142
|
+
const assertion = { ...relationship, provenance, extractor: routingExtractorIdentity,
|
|
143
|
+
confidence: 1, authority: 'inferred', sourceAuthority: 'inferred',
|
|
144
|
+
extractionReview: 'deterministic' };
|
|
145
|
+
facts.push({ ...assertion, id: sha256(stableJson(assertion)), snapshotId: context.snapshotId });
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
return { facts: facts.sort((a, b) => a.id.localeCompare(b.id)),
|
|
149
|
+
diagnostics: diagnostics.sort((a, b) => a.id.localeCompare(b.id)),
|
|
150
|
+
referencedPaths: [...referencedPaths].sort() };
|
|
151
|
+
}
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
import type { ScipSourceProject } from '../configuration/scip-project.js';
|
|
2
|
+
/** Only writes compiler configuration into a disposable committed tree; never source or dependencies. */
|
|
3
|
+
export declare function prepareScipSourceProject(root: string, project: ScipSourceProject, trackedPaths: readonly string[]): Promise<string>;
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { writeFile } from 'node:fs/promises';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { matchesRepositoryGlob } from '../repository/glob.js';
|
|
4
|
+
const generatedProject = '.weftlet-scip-source-project.json';
|
|
5
|
+
/** Only writes compiler configuration into a disposable committed tree; never source or dependencies. */
|
|
6
|
+
export async function prepareScipSourceProject(root, project, trackedPaths) {
|
|
7
|
+
if (trackedPaths.includes(generatedProject))
|
|
8
|
+
throw new Error(`Reserved SCIP project path is already tracked: ${generatedProject}`);
|
|
9
|
+
const files = trackedPaths.filter((path) => /\.(?:[cm]?ts|tsx)$/.test(path) &&
|
|
10
|
+
project.include.some((pattern) => matchesRepositoryGlob(path, pattern))).sort();
|
|
11
|
+
if (!files.length || files.length > 5000)
|
|
12
|
+
throw new Error('SCIP source project must select 1–5000 committed TypeScript files');
|
|
13
|
+
const tracked = new Set(trackedPaths);
|
|
14
|
+
const paths = Object.create(null);
|
|
15
|
+
for (const [name, path] of Object.entries(project.aliases ?? {}).sort(([a], [b]) => a.localeCompare(b))) {
|
|
16
|
+
if (!tracked.has(path))
|
|
17
|
+
throw new Error(`SCIP source alias ${name} targets an absent committed file: ${path}`);
|
|
18
|
+
if (!files.includes(path))
|
|
19
|
+
throw new Error(`SCIP source alias ${name} target must be selected by sourceProject.include: ${path}`);
|
|
20
|
+
paths[name] = [`./${path}`];
|
|
21
|
+
}
|
|
22
|
+
// A separate program deliberately avoids inherited build exclusions, output files,
|
|
23
|
+
// external tsconfig packages and machine-specific path mappings. This is local-symbol
|
|
24
|
+
// analysis, not a replacement for the repository's compiler or dependency installation.
|
|
25
|
+
await writeFile(join(root, generatedProject), JSON.stringify({
|
|
26
|
+
compilerOptions: {
|
|
27
|
+
target: 'ES2022', module: 'NodeNext', moduleResolution: 'NodeNext', jsx: 'preserve',
|
|
28
|
+
noEmit: true, skipLibCheck: true, types: [], baseUrl: '.', paths,
|
|
29
|
+
},
|
|
30
|
+
files: files.map((path) => `./${path}`), include: [], exclude: [],
|
|
31
|
+
}) + '\n', { flag: 'wx' });
|
|
32
|
+
return generatedProject;
|
|
33
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { spawn } from 'node:child_process';
|
|
1
|
+
import { execFile, spawn } from 'node:child_process';
|
|
2
|
+
import { join } from 'node:path';
|
|
2
3
|
export class SemanticProviderError extends Error {
|
|
3
4
|
code;
|
|
4
5
|
retryable;
|
|
@@ -25,8 +26,17 @@ export function runSemanticProcess(command, args, cwd, prompt, environment, opti
|
|
|
25
26
|
try {
|
|
26
27
|
if (process.platform !== 'win32' && child.pid)
|
|
27
28
|
process.kill(-child.pid, signal);
|
|
28
|
-
else
|
|
29
|
-
child.
|
|
29
|
+
else if (child.pid) {
|
|
30
|
+
// npm's Windows launcher starts a native child. Terminate its tree, not just the Node wrapper.
|
|
31
|
+
const systemRoot = process.env.SystemRoot ?? process.env.SYSTEMROOT;
|
|
32
|
+
const taskkill = systemRoot ? join(systemRoot, 'System32', 'taskkill.exe') : 'taskkill.exe';
|
|
33
|
+
execFile(taskkill, ['/PID', String(child.pid), '/T', '/F'], { windowsHide: true, timeout: 5000 }, error => {
|
|
34
|
+
if (error && child.exitCode === null && child.signalCode === null) {
|
|
35
|
+
failure = new SemanticProviderError('unavailable', 'Could not terminate the semantic provider process tree');
|
|
36
|
+
child.kill('SIGKILL');
|
|
37
|
+
}
|
|
38
|
+
});
|
|
39
|
+
}
|
|
30
40
|
}
|
|
31
41
|
catch { /* The process may have already exited. */ }
|
|
32
42
|
};
|
|
@@ -35,8 +45,10 @@ export function runSemanticProcess(command, args, cwd, prompt, environment, opti
|
|
|
35
45
|
return;
|
|
36
46
|
failure = new SemanticProviderError(code, `Semantic extraction ${code}`);
|
|
37
47
|
kill('SIGTERM');
|
|
38
|
-
|
|
39
|
-
|
|
48
|
+
if (process.platform !== 'win32') {
|
|
49
|
+
killTimer = setTimeout(() => kill('SIGKILL'), 1000);
|
|
50
|
+
killTimer.unref();
|
|
51
|
+
}
|
|
40
52
|
};
|
|
41
53
|
const cancel = () => stop('cancelled');
|
|
42
54
|
options.signal?.addEventListener('abort', cancel, { once: true });
|
|
@@ -4,10 +4,26 @@ import { tmpdir } from 'node:os';
|
|
|
4
4
|
import { join } from 'node:path';
|
|
5
5
|
import { prepareCodexAuthentication, readCodexLoginMode } from './codex-auth.js';
|
|
6
6
|
export { semanticCommandEnvironment } from './codex-auth.js';
|
|
7
|
+
import { resolveCodexInvocation } from './codex-command.js';
|
|
7
8
|
import { runSemanticProcess, SemanticProviderError } from "./semantic-process.js";
|
|
8
9
|
export { SemanticProviderError } from "./semantic-process.js";
|
|
9
10
|
import { sha256, stableJson } from "../infrastructure/util.js";
|
|
11
|
+
import { canonicalizeRepositoryDocumentNode, repositoryNodeIdentityVersion } from './repository-nodes.js';
|
|
10
12
|
import { assertSemanticCitation, numberedSemanticSource, semanticCitationSchema, } from "./semantic-citations.js";
|
|
13
|
+
function canonicalizeRepositoryDocuments(output) {
|
|
14
|
+
const ids = new Map();
|
|
15
|
+
const assertions = output.assertions.map((assertion) => {
|
|
16
|
+
const subject = canonicalizeRepositoryDocumentNode(assertion.subject);
|
|
17
|
+
const object = canonicalizeRepositoryDocumentNode(assertion.object);
|
|
18
|
+
ids.set(assertion.subject.id, subject.id);
|
|
19
|
+
ids.set(assertion.object.id, object.id);
|
|
20
|
+
return { ...assertion, subject, object };
|
|
21
|
+
});
|
|
22
|
+
return { ...output, assertions, diagnostics: output.diagnostics?.map((diagnostic) => ({
|
|
23
|
+
...diagnostic,
|
|
24
|
+
relatedNodeIds: diagnostic.relatedNodeIds.map((id) => ids.get(id) ?? id),
|
|
25
|
+
})) };
|
|
26
|
+
}
|
|
11
27
|
function omlEscape(value) {
|
|
12
28
|
return value.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\r?\n/g, '\\n');
|
|
13
29
|
}
|
|
@@ -75,7 +91,17 @@ function renderOml(key, assertions) {
|
|
|
75
91
|
'}',
|
|
76
92
|
].join('\n');
|
|
77
93
|
}
|
|
78
|
-
const spawnSemanticCommand = {
|
|
94
|
+
const spawnSemanticCommand = {
|
|
95
|
+
async run(command, args, cwd, prompt, environment, options) {
|
|
96
|
+
const started = performance.now();
|
|
97
|
+
const timeoutMs = options?.timeoutMs ?? 120_000;
|
|
98
|
+
const invocation = await resolveCodexInvocation(command, environment, process.platform, options);
|
|
99
|
+
const remaining = Math.ceil(timeoutMs - (performance.now() - started));
|
|
100
|
+
if (remaining <= 0)
|
|
101
|
+
throw new SemanticProviderError('timeout', 'Semantic extraction timeout');
|
|
102
|
+
return runSemanticProcess(invocation.command, [...invocation.prefixArgs, ...args], cwd, prompt, environment, { ...options, timeoutMs: remaining });
|
|
103
|
+
},
|
|
104
|
+
};
|
|
79
105
|
const kinds = [
|
|
80
106
|
'Project',
|
|
81
107
|
'Document',
|
|
@@ -202,14 +228,14 @@ export class CodexCliSemanticExtractor {
|
|
|
202
228
|
this.config = config;
|
|
203
229
|
this.runner = runner;
|
|
204
230
|
this.loginProbe = loginProbe;
|
|
205
|
-
this.identity = `codex-cli:${config.model}:${config.reasoningEffort}:${config.promptVersion}:contract-
|
|
231
|
+
this.identity = `codex-cli:${config.model}:${config.reasoningEffort}:${config.promptVersion}:contract-v3`;
|
|
206
232
|
}
|
|
207
233
|
async extract(source, context, options = {}) {
|
|
208
234
|
options.signal?.throwIfAborted();
|
|
209
235
|
if (Buffer.byteLength(source.content) > (this.config.maxSourceBytes ?? 65_536)) {
|
|
210
236
|
throw new SemanticProviderError('invalid-output', 'Semantic source exceeds configured byte budget');
|
|
211
237
|
}
|
|
212
|
-
const command =
|
|
238
|
+
const command = 'codex';
|
|
213
239
|
const environment = await prepareCodexAuthentication(this.config.auth, process.env, command, options, this.loginProbe);
|
|
214
240
|
const runRoot = await mkdtemp(join(tmpdir(), 'context-graph-codex-'));
|
|
215
241
|
const schemaPath = join(runRoot, 'schema.json');
|
|
@@ -224,7 +250,7 @@ export class CodexCliSemanticExtractor {
|
|
|
224
250
|
'Use neutral graph kinds and the allowed precise predicates. Never guess a missing mapping.',
|
|
225
251
|
'Use product-wide canonical IDs derived from durable concepts, not IDs scoped to this file.',
|
|
226
252
|
'Reuse capability, policy, workflow, contract, and component IDs that another source would use for the same concept.',
|
|
227
|
-
'
|
|
253
|
+
'For a repository document path ending in .md, .mdx or .txt, use document:<path> with kind Document. For other repository files use component:file:<path>; test/spec files have kind Evidence, other files Component. Use exact repository-relative paths; keep business concepts in their own IDs instead of assigning their kind to a file.',
|
|
228
254
|
'Capture why a rule exists, its decision authority, and cross-domain dependencies when the source explicitly states them.',
|
|
229
255
|
'Every assertion must cite inclusive 1-based source line numbers and carry calibrated confidence.',
|
|
230
256
|
`Source lines are explicitly prefixed N| below. Cite those original-source numbers, not prompt lines or visual wraps; valid lines are 1-${lineCount}.`,
|
|
@@ -319,6 +345,7 @@ export class SemanticHydrator {
|
|
|
319
345
|
failOnUnresolved: this.config.failOnUnresolved,
|
|
320
346
|
forceRevision,
|
|
321
347
|
diagnosticContractVersion: 3,
|
|
348
|
+
repositoryNodeIdentityVersion,
|
|
322
349
|
};
|
|
323
350
|
const key = sha256(stableJson(keyInput));
|
|
324
351
|
const existing = await this.store.read(key);
|
|
@@ -329,7 +356,7 @@ export class SemanticHydrator {
|
|
|
329
356
|
if (typeof extraction.rawModelOutput !== 'string' || Buffer.byteLength(extraction.rawModelOutput) > 2 * 1024 * 1024) {
|
|
330
357
|
throw new SemanticProviderError('invalid-output', 'Invalid or oversized raw semantic output');
|
|
331
358
|
}
|
|
332
|
-
const output = extraction.parsed;
|
|
359
|
+
const output = canonicalizeRepositoryDocuments(extraction.parsed);
|
|
333
360
|
const lineCount = source.content.split('\n').length;
|
|
334
361
|
const facts = output.assertions.map((assertion, index) => {
|
|
335
362
|
assertSemanticCitation(assertion.citation, source.path, lineCount, `assertion[${index}]`);
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { ContextGraphConfig } from "../configuration/config.js";
|
|
2
2
|
import type { ExtractionContext, FactExtractor, GraphFact } from "@weftlet/core";
|
|
3
|
-
import
|
|
3
|
+
import { GitRepository } from "../repository/git.js";
|
|
4
4
|
export interface ScipIndexResult {
|
|
5
5
|
readonly path: string;
|
|
6
6
|
readonly digest: string;
|
|
@@ -27,13 +27,17 @@ export declare class ScipSymbolExtractor implements FactExtractor {
|
|
|
27
27
|
constructor(repository: GitRepository, index: ScipIndexResult);
|
|
28
28
|
extract(context: ExtractionContext): Promise<readonly GraphFact[]>;
|
|
29
29
|
}
|
|
30
|
+
export declare const repositoryFileExtractorIdentity: {
|
|
31
|
+
readonly id: "repository-file-extractor";
|
|
32
|
+
readonly version: "2.0.0";
|
|
33
|
+
};
|
|
30
34
|
export declare class RepositoryFileExtractor implements FactExtractor {
|
|
31
35
|
private readonly repository;
|
|
32
36
|
private readonly roots;
|
|
33
37
|
private readonly scipDigest;
|
|
34
38
|
readonly identity: {
|
|
35
|
-
id:
|
|
36
|
-
version:
|
|
39
|
+
readonly id: "repository-file-extractor";
|
|
40
|
+
readonly version: "2.0.0";
|
|
37
41
|
};
|
|
38
42
|
constructor(repository: GitRepository, roots: readonly string[], scipDigest: string);
|
|
39
43
|
extract(context: ExtractionContext): Promise<readonly GraphFact[]>;
|