agentera 3.0.0-dev.72 → 3.0.0-dev.75
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +111 -0
- package/bundle/.agentera-build-source.json +8 -0
- package/bundle/CHANGELOG.md +17 -17
- package/bundle/extract-corpus-parity.json +1 -1
- package/bundle/references/adapters/package-publication.json +49 -53
- package/bundle/references/adapters/package-registry.yaml +5 -1
- package/bundle/references/adapters/package-surface-characterization.md +6 -4
- package/bundle/references/analysis/evidence-tier-authority.yaml +45 -4
- package/bundle/references/analysis/personal-glossary-evaluation-authority.yaml +175 -0
- package/bundle/references/analysis/personal-glossary-evaluation-corpus.yaml +291 -0
- package/bundle/references/analysis/personal-glossary-holdout.yaml +192 -0
- package/bundle/references/analysis/verification-policy.yaml +8 -13
- package/bundle/references/artifacts/glossary-entry-contract.yaml +1533 -35
- package/bundle/references/cli/update-channels.yaml +2 -2
- package/bundle/references/cli/vocabulary.md +1 -1
- package/bundle/references/meta/retained-reference-authority.yaml +16 -1
- package/bundle/skills/agentera/capabilities/profile/schemas/artifacts.yaml +11 -7
- package/bundle/skills/agentera/capabilities/profile/schemas/exit.yaml +6 -3
- package/bundle/skills/agentera/capabilities/profile/schemas/validation.yaml +13 -7
- package/bundle/skills/agentera/schemas/artifacts/glossary.yaml +5 -0
- package/bundle/skills/agentera/schemas/artifacts/plan.yaml +7 -0
- package/dist/.agentera-build-source.json +8 -0
- package/dist/analytics/extractCorpus/copilotSessions.js +13 -4
- package/dist/analytics/extractCorpus/core.js +223 -3
- package/dist/analytics/extractCorpus/corpus.js +18 -1
- package/dist/analytics/extractCorpus/cursorSessions.js +22 -4
- package/dist/analytics/extractCorpus/evidenceTiers.js +123 -17
- package/dist/analytics/extractCorpus/filesystemSources.js +4 -0
- package/dist/analytics/extractCorpus/index.js +2 -2
- package/dist/analytics/extractCorpus/jsonlSessions.js +52 -3
- package/dist/analytics/extractCorpus/sqliteSessions.js +12 -3
- package/dist/analytics/personalGlossaryAdmission.js +26 -46
- package/dist/analytics/personalGlossaryCandidateProjection.js +692 -0
- package/dist/analytics/personalGlossaryCandidateProjectionExcerpts.js +144 -0
- package/dist/analytics/personalGlossaryCandidateReadView.js +23 -0
- package/dist/analytics/personalGlossaryCurrentGeneration.js +25 -0
- package/dist/analytics/personalGlossaryDecision.js +203 -0
- package/dist/analytics/personalGlossaryExplicit.js +788 -0
- package/dist/analytics/personalGlossaryExplicitMining.js +285 -0
- package/dist/analytics/personalGlossaryExplicitScope.js +41 -0
- package/dist/analytics/personalGlossaryExplicitSegments.js +418 -0
- package/dist/analytics/personalGlossaryExplicitTypes.js +26 -0
- package/dist/analytics/personalGlossaryLexicalClassifier.js +175 -0
- package/dist/analytics/personalGlossaryProfile.js +13 -3
- package/dist/analytics/personalGlossaryRecurrence.js +618 -0
- package/dist/analytics/personalGlossaryReviewRecordStorage.js +428 -0
- package/dist/analytics/personalGlossaryReviewRecords.js +332 -0
- package/dist/capabilities/index.js +5 -1
- package/dist/capabilities/profile/instructions.js +50 -15
- package/dist/cli/commands/personalGlossaryCandidateReads.js +744 -0
- package/dist/cli/commands/personalGlossaryDecision.js +349 -0
- package/dist/cli/commands/personalGlossaryPublish.js +484 -0
- package/dist/cli/commands/personalGlossaryReviewRecordReads.js +209 -0
- package/dist/cli/commands/personalGlossaryReviewRecords.js +623 -0
- package/dist/cli/commands/prime/briefOrientation.js +118 -19
- package/dist/cli/commands/report.js +18 -2
- package/dist/cli/commands/schema.js +179 -0
- package/dist/cli/commands/verify.js +51 -9
- package/dist/cli/dispatch/lifecycle.js +51 -6
- package/dist/cli/help.js +48 -2
- package/dist/cli/migrationRequired.js +4 -1
- package/dist/core/atomicWriter.js +4 -1
- package/dist/core/developmentInvocation.js +4 -1
- package/dist/core/yaml.js +53 -1
- package/dist/eval/glossaryEvaluation.js +751 -0
- package/dist/eval/glossaryEvaluationProcess.js +29 -0
- package/dist/eval/glossaryEvaluationRunner.js +571 -0
- package/dist/eval/glossaryEvaluationSuccessReport.js +297 -0
- package/dist/registries/activationTuples.js +31 -3
- package/dist/registries/evidenceTierContract.js +4 -0
- package/dist/registries/explicitSegmentGrammarContract.js +619 -0
- package/dist/registries/glossaryCandidateContractPaths.js +6 -0
- package/dist/registries/glossaryCandidateContracts.js +820 -0
- package/dist/registries/glossaryCandidateDecisionAuthority.js +119 -0
- package/dist/registries/glossaryCandidateDecisionContract.js +62 -0
- package/dist/registries/glossaryCandidateProjectionAuthority.js +197 -0
- package/dist/registries/glossaryCandidateProjectionContract.js +104 -0
- package/dist/registries/glossaryEntryContract.js +105 -103
- package/dist/registries/glossaryEntryTemporal.js +12 -0
- package/dist/registries/glossaryMiningAuthority.js +764 -0
- package/dist/registries/glossaryProfileFullContract.js +24 -0
- package/dist/registries/glossaryReviewRecordsAuthority.js +426 -0
- package/dist/registries/glossaryReviewRecordsContract.js +129 -0
- package/dist/registries/glossaryTermIdentity.js +122 -0
- package/dist/registries/packagePublication.js +3 -3
- package/dist/registries/personalGlossaryContracts.js +1 -0
- package/dist/release/releaseMetadata.js +15 -8
- package/dist/upgrade/npxPlatformStatus.js +2 -1
- package/dist/upgrade/projectIntegration.js +7 -1
- package/dist/validate/activationArtifactEvidence.js +18 -4
- package/dist/validate/activationEvidenceManifest.js +101 -22
- package/package.json +2 -4
- package/dist/cli/commands/personalGlossary.js +0 -176
|
@@ -2,7 +2,7 @@ import fs from "node:fs";
|
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import { pyJsonIndentSorted } from "../../core/pyjson.js";
|
|
4
4
|
import { writeFileAtomic } from "../../core/atomicWriter.js";
|
|
5
|
-
import { stableId, defaultProfileDir } from "./core.js";
|
|
5
|
+
import { assessProvenance, stableId, defaultProfileDir, } from "./core.js";
|
|
6
6
|
import { evidenceTierBounds, loadEvidenceTierContract } from "../../registries/evidenceTierContract.js";
|
|
7
7
|
/**
|
|
8
8
|
* Bounded tier publication and direct retrieval (plan Task 2).
|
|
@@ -87,8 +87,16 @@ export function deriveSignalRecords(fullRecords) {
|
|
|
87
87
|
project_id: asStringField(r, "project_id"),
|
|
88
88
|
runtime: typeof r.runtime === "string" ? r.runtime : null,
|
|
89
89
|
source_product: asStringField(r, "source_product"),
|
|
90
|
+
source_class: asStringField(r, "source_class"),
|
|
90
91
|
evidence_anchor: sourceId,
|
|
92
|
+
active_runtime: r.active_runtime === true,
|
|
91
93
|
};
|
|
94
|
+
if (typeof r.origin_id === "string")
|
|
95
|
+
rec.origin_id = r.origin_id;
|
|
96
|
+
if (typeof r.content_fingerprint === "string")
|
|
97
|
+
rec.content_fingerprint = r.content_fingerprint;
|
|
98
|
+
if (typeof r.author_class === "string")
|
|
99
|
+
rec.author_class = r.author_class;
|
|
92
100
|
if (typeof r.conversation_key === "string")
|
|
93
101
|
rec.conversation_key = r.conversation_key;
|
|
94
102
|
if (typeof r.session_id === "string")
|
|
@@ -280,6 +288,9 @@ function coverageFromStatuses(runtimeStatuses) {
|
|
|
280
288
|
}
|
|
281
289
|
if (GAP.has(status)) {
|
|
282
290
|
const entry = { runtime, reason: reason || status };
|
|
291
|
+
if (Array.isArray(s.provenance_missing_fields) && s.provenance_missing_fields.length > 0) {
|
|
292
|
+
entry.reason += ` (missing provenance: ${s.provenance_missing_fields.join(",")})`;
|
|
293
|
+
}
|
|
283
294
|
if (typeof s.store_path === "string")
|
|
284
295
|
entry.store_path = s.store_path;
|
|
285
296
|
skipped.push(entry);
|
|
@@ -297,6 +308,31 @@ export function publishEvidenceTiers(records, opts, contractPath) {
|
|
|
297
308
|
const bounds = evidenceTierBounds(contractPath);
|
|
298
309
|
const productFamily = productFamilyIndex(contractPath);
|
|
299
310
|
const sortedRecords = records.slice().sort(tierSortKey);
|
|
311
|
+
const provenance = assessProvenance(sortedRecords);
|
|
312
|
+
const runtimeStatuses = [...(opts.runtimeStatuses ?? [])];
|
|
313
|
+
if (!provenance.complete) {
|
|
314
|
+
const gapProducts = new Map();
|
|
315
|
+
for (const record of sortedRecords) {
|
|
316
|
+
if (Object.keys(record).length === 0)
|
|
317
|
+
continue;
|
|
318
|
+
if (assessProvenance([record]).complete)
|
|
319
|
+
continue;
|
|
320
|
+
const product = typeof record.source_product === "string" ? record.source_product : "unknown";
|
|
321
|
+
if (!gapProducts.has(product)) {
|
|
322
|
+
gapProducts.set(product, {
|
|
323
|
+
runtime: typeof record.runtime === "string" ? record.runtime : null,
|
|
324
|
+
source_product: product,
|
|
325
|
+
source_class: typeof record.source_class === "string" ? record.source_class : "unknown",
|
|
326
|
+
active_runtime: record.active_runtime === true,
|
|
327
|
+
status: "degraded",
|
|
328
|
+
reason: "provenance_missing",
|
|
329
|
+
provenance_missing_fields: provenance.missingFields,
|
|
330
|
+
provenance_missing_records: provenance.missingRecords,
|
|
331
|
+
});
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
runtimeStatuses.push(...gapProducts.values());
|
|
335
|
+
}
|
|
300
336
|
const signals = deriveSignalRecords(sortedRecords);
|
|
301
337
|
const { records: selectedSignals, selection } = selectSignalsForBound(signals, bounds.signalByteCap);
|
|
302
338
|
const shards = shardFullEvidence(sortedRecords, bounds.shardByteCap, productFamily);
|
|
@@ -342,7 +378,7 @@ export function publishEvidenceTiers(records, opts, contractPath) {
|
|
|
342
378
|
families: familiesAgg,
|
|
343
379
|
shards: shardDescriptors,
|
|
344
380
|
signal: { path: signalRel, record_count: selectedSignals.length, bytes: signalBytes, selection },
|
|
345
|
-
coverage: coverageFromStatuses(
|
|
381
|
+
coverage: coverageFromStatuses(runtimeStatuses),
|
|
346
382
|
corpus_metadata: opts.corpusMetadata,
|
|
347
383
|
};
|
|
348
384
|
writeFileAtomic(path.join(stageDir, "manifest.json"), encodeSorted(manifest) + "\n");
|
|
@@ -433,6 +469,68 @@ function isEvidenceTierManifest(value) {
|
|
|
433
469
|
Array.isArray(shard.source_ids) &&
|
|
434
470
|
shard.source_ids.every((sourceId) => typeof sourceId === "string"));
|
|
435
471
|
}
|
|
472
|
+
function lowerSha256(value) {
|
|
473
|
+
return typeof value === "string" && /^[a-f0-9]{64}$/.test(value);
|
|
474
|
+
}
|
|
475
|
+
/** Validate bounded signal provenance without reading any full-evidence shard. */
|
|
476
|
+
export function missingSignalProvenanceFields(record) {
|
|
477
|
+
const missing = [];
|
|
478
|
+
const required = [
|
|
479
|
+
["source_id", typeof record.source_id === "string" && record.source_id.length > 0],
|
|
480
|
+
["source_kind", typeof record.source_kind === "string" && record.source_kind.length > 0],
|
|
481
|
+
["signal_type", typeof record.signal_type === "string" && record.signal_type.length > 0],
|
|
482
|
+
["timestamp", typeof record.timestamp === "string" && record.timestamp.length > 0],
|
|
483
|
+
["project_id", typeof record.project_id === "string" && record.project_id.length > 0],
|
|
484
|
+
["source_product", typeof record.source_product === "string" && record.source_product.length > 0],
|
|
485
|
+
["active_runtime", typeof record.active_runtime === "boolean"],
|
|
486
|
+
["source_class", typeof record.source_class === "string" && record.source_class.length > 0],
|
|
487
|
+
["evidence_anchor", record.evidence_anchor === record.source_id && record.evidence_anchor.length > 0],
|
|
488
|
+
];
|
|
489
|
+
for (const [field, present] of required)
|
|
490
|
+
if (!present)
|
|
491
|
+
missing.push(field);
|
|
492
|
+
if (CONVERSATION_SIGNAL_SOURCE_KINDS.has(record.source_kind)) {
|
|
493
|
+
if (!lowerSha256(record.origin_id))
|
|
494
|
+
missing.push("origin_id");
|
|
495
|
+
if (!lowerSha256(record.content_fingerprint))
|
|
496
|
+
missing.push("content_fingerprint");
|
|
497
|
+
if (typeof record.author_class !== "string" || record.author_class.length === 0)
|
|
498
|
+
missing.push("author_class");
|
|
499
|
+
if (typeof record.session_id !== "string" || record.session_id.length === 0)
|
|
500
|
+
missing.push("session_id");
|
|
501
|
+
}
|
|
502
|
+
return [...new Set(missing)];
|
|
503
|
+
}
|
|
504
|
+
const CONVERSATION_SIGNAL_SOURCE_KINDS = new Set(["conversation_turn", "history_prompt"]);
|
|
505
|
+
function signalProvenanceGaps(records) {
|
|
506
|
+
const gaps = new Set();
|
|
507
|
+
for (const record of records) {
|
|
508
|
+
if (!isMapping(record)) {
|
|
509
|
+
gaps.add("record");
|
|
510
|
+
continue;
|
|
511
|
+
}
|
|
512
|
+
for (const field of missingSignalProvenanceFields(record))
|
|
513
|
+
gaps.add(field);
|
|
514
|
+
}
|
|
515
|
+
return [...gaps].sort();
|
|
516
|
+
}
|
|
517
|
+
function readSignalPayload(gen) {
|
|
518
|
+
const signalPath = path.join(gen.dir, gen.manifest.signal.path);
|
|
519
|
+
if (!fs.existsSync(signalPath))
|
|
520
|
+
return null;
|
|
521
|
+
try {
|
|
522
|
+
const bytes = fs.statSync(signalPath).size;
|
|
523
|
+
if (bytes > evidenceTierBounds().signalByteCap)
|
|
524
|
+
return null;
|
|
525
|
+
const data = JSON.parse(fs.readFileSync(signalPath, "utf-8"));
|
|
526
|
+
if (!data || !Array.isArray(data.records))
|
|
527
|
+
return null;
|
|
528
|
+
return { records: data.records, bytes };
|
|
529
|
+
}
|
|
530
|
+
catch {
|
|
531
|
+
return null;
|
|
532
|
+
}
|
|
533
|
+
}
|
|
436
534
|
/** Resolve the current generation directory and manifest, or null. */
|
|
437
535
|
export function readCurrentGeneration(tiersDir) {
|
|
438
536
|
const pointer = readCurrentPointer(tiersDir);
|
|
@@ -453,26 +551,16 @@ export function readCurrentGeneration(tiersDir) {
|
|
|
453
551
|
}
|
|
454
552
|
}
|
|
455
553
|
/** Read the bounded signal tier records for the current generation. */
|
|
456
|
-
export function readSignalTier(tiersDir) {
|
|
554
|
+
export function readSignalTier(tiersDir, options = {}) {
|
|
457
555
|
const gen = readCurrentGeneration(tiersDir);
|
|
458
556
|
if (!gen)
|
|
459
557
|
return null;
|
|
460
|
-
const
|
|
461
|
-
if (!
|
|
558
|
+
const payload = readSignalPayload(gen);
|
|
559
|
+
if (!payload)
|
|
462
560
|
return null;
|
|
463
|
-
|
|
464
|
-
const bytes = fs.statSync(signalPath).size;
|
|
465
|
-
const signalByteCap = evidenceTierBounds().signalByteCap;
|
|
466
|
-
if (bytes > signalByteCap)
|
|
467
|
-
return null;
|
|
468
|
-
const data = JSON.parse(fs.readFileSync(signalPath, "utf-8"));
|
|
469
|
-
if (!data || !Array.isArray(data.records))
|
|
470
|
-
return null;
|
|
471
|
-
return { records: data.records, bytes, manifest: gen.manifest };
|
|
472
|
-
}
|
|
473
|
-
catch {
|
|
561
|
+
if (!options.allowProvenanceGaps && signalProvenanceGaps(payload.records).length > 0)
|
|
474
562
|
return null;
|
|
475
|
-
}
|
|
563
|
+
return { ...payload, manifest: gen.manifest };
|
|
476
564
|
}
|
|
477
565
|
/** Resolve a bounded signal's identity (evidence_anchor) to its full record. */
|
|
478
566
|
export function resolveEvidenceAnchor(anchor, tiersDir) {
|
|
@@ -543,6 +631,24 @@ export function evidenceTierCompatibility(tiersDir, corpusPath) {
|
|
|
543
631
|
gen.manifest.signal.bytes > evidenceTierBounds().signalByteCap) {
|
|
544
632
|
return { state: "oversized", reason: "oversized", artifact: gen.manifest.signal.path };
|
|
545
633
|
}
|
|
634
|
+
let signalPayload;
|
|
635
|
+
try {
|
|
636
|
+
const data = JSON.parse(fs.readFileSync(signalPath, "utf-8"));
|
|
637
|
+
signalPayload = data && Array.isArray(data.records) ? { records: data.records, bytes: actualSignalBytes } : null;
|
|
638
|
+
}
|
|
639
|
+
catch {
|
|
640
|
+
signalPayload = null;
|
|
641
|
+
}
|
|
642
|
+
if (!signalPayload)
|
|
643
|
+
return { state: "corrupt", reason: "unreadable_or_schema_divergent" };
|
|
644
|
+
const provenanceGaps = signalProvenanceGaps(signalPayload.records);
|
|
645
|
+
if (provenanceGaps.length > 0) {
|
|
646
|
+
return {
|
|
647
|
+
state: "incomplete",
|
|
648
|
+
reason: "provenance_missing",
|
|
649
|
+
detail: provenanceGaps.map((field) => `signal: missing ${field}`),
|
|
650
|
+
};
|
|
651
|
+
}
|
|
546
652
|
// Incomplete: a supported source family is skipped/sparse/missing.
|
|
547
653
|
const skipped = gen.manifest.coverage.skipped;
|
|
548
654
|
if (skipped.length > 0) {
|
|
@@ -23,7 +23,9 @@ export function extractInstructionDocuments(projectRoots, errors) {
|
|
|
23
23
|
timestamp: isoFromMtime(p),
|
|
24
24
|
projectPath: root,
|
|
25
25
|
runtime: "filesystem",
|
|
26
|
+
origin: resolvePath(p),
|
|
26
27
|
sourceParts: [resolvePath(p)],
|
|
28
|
+
content,
|
|
27
29
|
data: { doc_type: docType, name: filename, content, scope: "project" },
|
|
28
30
|
}));
|
|
29
31
|
}
|
|
@@ -109,7 +111,9 @@ export function extractProjectConfigSignals(projectRoots, errors) {
|
|
|
109
111
|
timestamp: isoFromMtime(p),
|
|
110
112
|
projectPath: root,
|
|
111
113
|
runtime: "filesystem",
|
|
114
|
+
origin: resolvePath(p),
|
|
112
115
|
sourceParts: [resolvePath(p), configType],
|
|
116
|
+
content: signals.join("\n"),
|
|
113
117
|
data: { config_type: configType, file_path: path.relative(root, p), signals },
|
|
114
118
|
}));
|
|
115
119
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { ADAPTER_VERSION, FAMILIES, RUNTIME_STORE_GLOBS, isoNow, isoFromMtime, stableId, projectIdFromPath, defaultAgenteraHome, defaultProfileDir, defaultOutputPath, runtimeStatus, discoverRuntimeStore, record, payloadItem, eventKind, eventTimestamp, textFromContent, claudeContentItems, iterJsonl, signalType, } from "./core.js";
|
|
1
|
+
export { ADAPTER_VERSION, FAMILIES, RUNTIME_STORE_GLOBS, isoNow, isoFromMtime, stableId, originIdentity, contentFingerprint, transportProvenance, authorClassForRole, missingProvenanceFields, assessProvenance, countIndependentOrigins, projectIdFromPath, defaultAgenteraHome, defaultProfileDir, defaultOutputPath, runtimeStatus, discoverRuntimeStore, record, payloadItem, eventKind, eventTimestamp, textFromContent, claudeContentItems, iterJsonl, signalType, } from "./core.js";
|
|
2
2
|
export { extractInstructionDocuments, extractProjectConfigSignals } from "./filesystemSources.js";
|
|
3
3
|
export { extractCodexSessions, extractClaudeProjectSessions } from "./jsonlSessions.js";
|
|
4
4
|
export { extractOpencodeSessions, PermissionDeniedError } from "./sqliteSessions.js";
|
|
@@ -8,7 +8,7 @@ export { ExtractionNotImplementedError, dedupeRecords, buildMetadata, buildCorpu
|
|
|
8
8
|
export { formatTruncationWarnings, resolveSqliteCaps, } from "./sqliteCaps.js";
|
|
9
9
|
export { COVERAGE_EXIT_FLAGGED, corpusEnvelopeCoverage, formatCoverageSummaryText, resolveRuntimeStoreConfigs, runCoverageAudit, } from "./coverageAudit.js";
|
|
10
10
|
export { parseExtractArgs, extractCorpusMain } from "./cli.js";
|
|
11
|
-
export { TIER_SCHEMA_VERSION, CURRENT_POINTER_VERSION, defaultTiersDir, familyOf, deriveSignalRecords, shardFullEvidence, selectSignalsForBound, generationId, publishEvidenceTiers, readCurrentPointer, readCurrentGeneration, readSignalTier, getFullRecord, resolveEvidenceAnchor, evidenceTierCompatibility, iterTierRecords, readTierCorpusMetadata, } from "./evidenceTiers.js";
|
|
11
|
+
export { TIER_SCHEMA_VERSION, CURRENT_POINTER_VERSION, defaultTiersDir, familyOf, deriveSignalRecords, shardFullEvidence, selectSignalsForBound, generationId, publishEvidenceTiers, readCurrentPointer, readCurrentGeneration, readSignalTier, missingSignalProvenanceFields, getFullRecord, resolveEvidenceAnchor, evidenceTierCompatibility, iterTierRecords, readTierCorpusMetadata, } from "./evidenceTiers.js";
|
|
12
12
|
export { tiersDirForCorpusPath, resolveTiersDir, isAnalyzable, recoveryForState, assessTiers, readBoundedMetadata, iterBoundedRecords, legacyCorpusReadable, } from "./tierReader.js";
|
|
13
13
|
export { EXTRACT_CORPUS_PARITY_SCHEMA, buildExtractCorpusParityManifest, opencodeParitySnapshot, } from "./extractCorpusParity.js";
|
|
14
14
|
export { readProfileSignals, resolveProfileEvidence, profileSignalsStatus, assessProfileSufficiency, } from "../profileSignals.js";
|
|
@@ -1,12 +1,43 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import { resolvePath } from "../../core/paths.js";
|
|
4
|
-
import { claudeContentItems, eventKind, eventTimestamp, isPlainObject, isoFromMtime, iterJsonl, payloadItem, record, rglob, signalType, textFromContent, toolCallRecord, toolCallRecordFromItem, } from "./core.js";
|
|
4
|
+
import { claudeContentItems, authorClassForRole, eventKind, eventTimestamp, inspectTransportProvenance, isPlainObject, isoFromMtime, iterJsonl, payloadItem, record, rglob, signalType, textFromContent, transportProvenance, toolCallRecord, toolCallRecordFromItem, } from "./core.js";
|
|
5
5
|
function pathStem(p) {
|
|
6
6
|
const base = path.basename(p);
|
|
7
7
|
const ext = path.extname(base);
|
|
8
8
|
return ext ? base.slice(0, -ext.length) : base;
|
|
9
9
|
}
|
|
10
|
+
function updateSessionProvenance(current, update) {
|
|
11
|
+
if (update.originState === "invalid")
|
|
12
|
+
return current;
|
|
13
|
+
const originId = update.originState === "valid" ? update.originId : current.originId;
|
|
14
|
+
const originChanged = update.originState === "valid" && originId !== current.originId;
|
|
15
|
+
if (originChanged) {
|
|
16
|
+
return {
|
|
17
|
+
originId,
|
|
18
|
+
authorClass: update.authorState === "valid" ? update.authorClass : null,
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
if (originId !== null && update.authorState === "valid") {
|
|
22
|
+
return { originId, authorClass: update.authorClass };
|
|
23
|
+
}
|
|
24
|
+
return { originId, authorClass: current.authorClass };
|
|
25
|
+
}
|
|
26
|
+
function codexMessageProvenance(role, message, session) {
|
|
27
|
+
const originId = message.originState === "valid" ? message.originId : session.originId;
|
|
28
|
+
if (message.originState === "invalid" || message.authorState === "invalid") {
|
|
29
|
+
return { originId, authorClass: null };
|
|
30
|
+
}
|
|
31
|
+
const explicitAuthorClass = message.authorState === "valid"
|
|
32
|
+
? message.authorClass
|
|
33
|
+
: originId !== null && originId === session.originId
|
|
34
|
+
? session.authorClass
|
|
35
|
+
: null;
|
|
36
|
+
return {
|
|
37
|
+
originId,
|
|
38
|
+
authorClass: authorClassForRole(role, explicitAuthorClass, originId),
|
|
39
|
+
};
|
|
40
|
+
}
|
|
10
41
|
export function extractCodexSessions(sessionsDir, errors) {
|
|
11
42
|
if (sessionsDir === null || !fs.existsSync(sessionsDir))
|
|
12
43
|
return [];
|
|
@@ -14,6 +45,7 @@ export function extractCodexSessions(sessionsDir, errors) {
|
|
|
14
45
|
for (const p of rglob(sessionsDir, "*.jsonl")) {
|
|
15
46
|
const fallbackTimestamp = isoFromMtime(p);
|
|
16
47
|
let sessionId = pathStem(p);
|
|
48
|
+
let sessionProvenance = { originId: null, authorClass: null };
|
|
17
49
|
let projectPath = null;
|
|
18
50
|
let previousAssistant = "";
|
|
19
51
|
let index = 0;
|
|
@@ -25,6 +57,7 @@ export function extractCodexSessions(sessionsDir, errors) {
|
|
|
25
57
|
const sid = payload.id || event.id;
|
|
26
58
|
if (typeof sid === "string" && sid)
|
|
27
59
|
sessionId = sid;
|
|
60
|
+
sessionProvenance = updateSessionProvenance(sessionProvenance, inspectTransportProvenance(event, payload));
|
|
28
61
|
const cwd = payload.cwd || payload.working_directory;
|
|
29
62
|
if (typeof cwd === "string" && cwd)
|
|
30
63
|
projectPath = cwd;
|
|
@@ -46,7 +79,7 @@ export function extractCodexSessions(sessionsDir, errors) {
|
|
|
46
79
|
let role = item.role || item.actor;
|
|
47
80
|
if (kind === "user_msg")
|
|
48
81
|
role = "user";
|
|
49
|
-
if (role !== "
|
|
82
|
+
if (typeof role !== "string" || role.length === 0)
|
|
50
83
|
continue;
|
|
51
84
|
const skipType = itemType !== undefined && itemType !== null && itemType !== "message";
|
|
52
85
|
if (skipType && !["response_item", "user_msg"].includes(kind))
|
|
@@ -55,6 +88,7 @@ export function extractCodexSessions(sessionsDir, errors) {
|
|
|
55
88
|
if (!content)
|
|
56
89
|
continue;
|
|
57
90
|
const timestamp = eventTimestamp(event, fallbackTimestamp);
|
|
91
|
+
const transport = codexMessageProvenance(role, inspectTransportProvenance(event, item), sessionProvenance);
|
|
58
92
|
const data = { actor: role, content };
|
|
59
93
|
if (role === "user") {
|
|
60
94
|
if (previousAssistant)
|
|
@@ -73,6 +107,9 @@ export function extractCodexSessions(sessionsDir, errors) {
|
|
|
73
107
|
runtime: "codex",
|
|
74
108
|
sourceParts: [resolvePath(p), index, role, content.slice(0, 80)],
|
|
75
109
|
sessionId,
|
|
110
|
+
originId: transport.originId,
|
|
111
|
+
authorClass: transport.authorClass,
|
|
112
|
+
content,
|
|
76
113
|
data,
|
|
77
114
|
}));
|
|
78
115
|
if (role === "user") {
|
|
@@ -85,6 +122,9 @@ export function extractCodexSessions(sessionsDir, errors) {
|
|
|
85
122
|
runtime: "codex",
|
|
86
123
|
sourceParts: [resolvePath(p), index, "history", content.slice(0, 120)],
|
|
87
124
|
sessionId,
|
|
125
|
+
originId: transport.originId,
|
|
126
|
+
authorClass: transport.authorClass,
|
|
127
|
+
content,
|
|
88
128
|
data: { prompt: content, signal_type: sig },
|
|
89
129
|
}));
|
|
90
130
|
}
|
|
@@ -152,12 +192,15 @@ export function extractClaudeProjectSessions(projectsDir, errors) {
|
|
|
152
192
|
if (isPlainObject(message))
|
|
153
193
|
role = message.role;
|
|
154
194
|
}
|
|
155
|
-
if (role !== "
|
|
195
|
+
if (typeof role !== "string" || role.length === 0)
|
|
156
196
|
continue;
|
|
157
197
|
const content = textFromContent(event.content || event.text || (isPlainObject(event.message) ? event.message : null));
|
|
158
198
|
if (!content)
|
|
159
199
|
continue;
|
|
160
200
|
const timestamp = eventTimestamp(event, fallbackTimestamp);
|
|
201
|
+
const transport = transportProvenance(event, isPlainObject(event.message) ? event.message : null);
|
|
202
|
+
const originId = transport.originId;
|
|
203
|
+
const authorClass = authorClassForRole(role, transport.authorClass, transport.originId);
|
|
161
204
|
const data = { actor: role, content };
|
|
162
205
|
if (role === "user") {
|
|
163
206
|
if (previousAssistant)
|
|
@@ -179,6 +222,9 @@ export function extractClaudeProjectSessions(projectsDir, errors) {
|
|
|
179
222
|
activeRuntime: false,
|
|
180
223
|
sourceParts: [resolvePath(p), index, role, content.slice(0, 80)],
|
|
181
224
|
sessionId,
|
|
225
|
+
originId,
|
|
226
|
+
authorClass,
|
|
227
|
+
content,
|
|
182
228
|
data,
|
|
183
229
|
}));
|
|
184
230
|
if (role === "user") {
|
|
@@ -194,6 +240,9 @@ export function extractClaudeProjectSessions(projectsDir, errors) {
|
|
|
194
240
|
activeRuntime: false,
|
|
195
241
|
sourceParts: [resolvePath(p), index, "history", content.slice(0, 120)],
|
|
196
242
|
sessionId,
|
|
243
|
+
originId,
|
|
244
|
+
authorClass,
|
|
245
|
+
content,
|
|
197
246
|
data: { prompt: content, signal_type: sig },
|
|
198
247
|
}));
|
|
199
248
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
2
|
import { createRequire } from "node:module";
|
|
3
3
|
import { resolvePath } from "../../core/paths.js";
|
|
4
|
-
import { isoFromMtime, record, signalType, textFromContent, } from "./core.js";
|
|
4
|
+
import { authorClassForRole, isoFromMtime, record, signalType, textFromContent, transportProvenance, } from "./core.js";
|
|
5
5
|
import { isPlainObject, isFilePath, rglob } from "./core.js";
|
|
6
6
|
import { resolveSqliteCaps, } from "./sqliteCaps.js";
|
|
7
7
|
const requireCjs = createRequire(import.meta.url);
|
|
@@ -271,8 +271,11 @@ export function extractOpencodeSessions(storePath, errors, ctx) {
|
|
|
271
271
|
const messageTime = row.message_time || nestedTimeCreated(messageData);
|
|
272
272
|
let item = messages.get(messageId);
|
|
273
273
|
if (item === undefined) {
|
|
274
|
+
const transport = transportProvenance(messageData);
|
|
274
275
|
item = {
|
|
275
276
|
role,
|
|
277
|
+
origin_id: transport.originId,
|
|
278
|
+
author_class: authorClassForRole(role, transport.authorClass, transport.originId),
|
|
276
279
|
session_id: String(row.session_id),
|
|
277
280
|
project_path: row.project_path,
|
|
278
281
|
timestamp: sqliteTimestamp(messageTime, fallbackTimestamp),
|
|
@@ -305,9 +308,9 @@ export function extractOpencodeSessions(storePath, errors, ctx) {
|
|
|
305
308
|
let index = 0;
|
|
306
309
|
for (const item of messages.values()) {
|
|
307
310
|
index += 1;
|
|
308
|
-
|
|
309
|
-
if (role !== "user" && role !== "assistant")
|
|
311
|
+
if (typeof item.role !== "string" || item.role.length === 0)
|
|
310
312
|
continue;
|
|
313
|
+
const role = item.role.toLowerCase();
|
|
311
314
|
const content = item.parts.filter((p) => p).join("\n");
|
|
312
315
|
if (!content && item.tools.length === 0)
|
|
313
316
|
continue;
|
|
@@ -331,6 +334,9 @@ export function extractOpencodeSessions(storePath, errors, ctx) {
|
|
|
331
334
|
runtime: "opencode",
|
|
332
335
|
sourceParts: [resolvePath(dbPath), index, role, content.slice(0, 80)],
|
|
333
336
|
sessionId: item.session_id,
|
|
337
|
+
originId: item.origin_id,
|
|
338
|
+
authorClass: item.author_class,
|
|
339
|
+
content,
|
|
334
340
|
data,
|
|
335
341
|
}));
|
|
336
342
|
}
|
|
@@ -357,6 +363,9 @@ export function extractOpencodeSessions(storePath, errors, ctx) {
|
|
|
357
363
|
runtime: "opencode",
|
|
358
364
|
sourceParts: [resolvePath(dbPath), index, "history", content.slice(0, 120)],
|
|
359
365
|
sessionId: item.session_id,
|
|
366
|
+
originId: item.origin_id,
|
|
367
|
+
authorClass: item.author_class,
|
|
368
|
+
content,
|
|
360
369
|
data: { prompt: content, signal_type: sig },
|
|
361
370
|
}));
|
|
362
371
|
}
|
|
@@ -1,34 +1,7 @@
|
|
|
1
1
|
import { personalGlossaryAdmissionContract } from "../registries/glossaryEntryContract.js";
|
|
2
2
|
import { readSignalTier, resolveEvidenceAnchor, } from "./extractCorpus/evidenceTiers.js";
|
|
3
3
|
import { assessTiers, recoveryForState } from "./extractCorpus/tierReader.js";
|
|
4
|
-
|
|
5
|
-
return value
|
|
6
|
-
.trim()
|
|
7
|
-
.replace(/[.!?]+$/, "")
|
|
8
|
-
.trim();
|
|
9
|
-
}
|
|
10
|
-
/** Classify only authority-supported explicit correction or clarification forms. */
|
|
11
|
-
export function classifyExplicitGlossaryLanguage(text) {
|
|
12
|
-
const correction = /\b(?:actually|not quite|instead|correction)\b[^`"']*([`"'])([^`"']+)\1\s+means\s+(.+)$/i.exec(text);
|
|
13
|
-
const clarification = /\bto clarify\b.*?\bprefer\s+([`"'])([^`"']+)\1\s+to mean\s+(.+)$/i.exec(text);
|
|
14
|
-
const match = correction ?? clarification;
|
|
15
|
-
if (!match)
|
|
16
|
-
return null;
|
|
17
|
-
const term = match[2].trim();
|
|
18
|
-
const meaning = trimMeaning(match[3]);
|
|
19
|
-
return term && meaning ? { term, meaning } : null;
|
|
20
|
-
}
|
|
21
|
-
function recordText(record) {
|
|
22
|
-
const data = record.data;
|
|
23
|
-
if (!data || typeof data !== "object" || Array.isArray(data))
|
|
24
|
-
return "";
|
|
25
|
-
const object = data;
|
|
26
|
-
for (const field of ["text", "prompt", "content"]) {
|
|
27
|
-
if (typeof object[field] === "string")
|
|
28
|
-
return object[field];
|
|
29
|
-
}
|
|
30
|
-
return "";
|
|
31
|
-
}
|
|
4
|
+
import { mineExplicitGlossaryCandidates } from "./personalGlossaryExplicitMining.js";
|
|
32
5
|
function semanticValues(record) {
|
|
33
6
|
const data = record.data;
|
|
34
7
|
if (!data || typeof data !== "object" || Array.isArray(data))
|
|
@@ -50,6 +23,10 @@ function containsCompleteTerm(text, term) {
|
|
|
50
23
|
const escaped = term.trim().replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
51
24
|
return escaped.length > 0 && new RegExp(`(^|[^a-z0-9])${escaped}(?=$|[^a-z0-9])`, "i").test(text);
|
|
52
25
|
}
|
|
26
|
+
function hasInferredProvenance(signal) {
|
|
27
|
+
return (/^[a-f0-9]{64}$/.test(signal.origin_id ?? "") &&
|
|
28
|
+
/^[a-f0-9]{64}$/.test(signal.content_fingerprint ?? ""));
|
|
29
|
+
}
|
|
53
30
|
function unavailable(state) {
|
|
54
31
|
return { state, status: "unavailable", candidates: [], recovery: recoveryForState(state) };
|
|
55
32
|
}
|
|
@@ -75,6 +52,8 @@ export function admitPersonalGlossaryEvidence(input, resolveAnchor = resolveEvid
|
|
|
75
52
|
inferredKinds.has(signal.source_kind)));
|
|
76
53
|
const resolved = new Map();
|
|
77
54
|
for (const signal of selected) {
|
|
55
|
+
if (explicitTypes.has(signal.signal_type))
|
|
56
|
+
continue;
|
|
78
57
|
if (resolved.has(signal.evidence_anchor))
|
|
79
58
|
continue;
|
|
80
59
|
const record = resolveAnchor(signal.evidence_anchor, input.tiersDir);
|
|
@@ -85,28 +64,24 @@ export function admitPersonalGlossaryEvidence(input, resolveAnchor = resolveEvid
|
|
|
85
64
|
}
|
|
86
65
|
}
|
|
87
66
|
const candidates = [];
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
continue;
|
|
91
|
-
const classified = classifyExplicitGlossaryLanguage(recordText(record));
|
|
92
|
-
if (!classified)
|
|
93
|
-
continue;
|
|
67
|
+
const explicitMining = mineExplicitGlossaryCandidates({ tiersDir: input.tiersDir }, resolveAnchor);
|
|
68
|
+
for (const { capsule } of explicitMining.candidates) {
|
|
94
69
|
candidates.push({
|
|
95
70
|
kind: "personal_explicit_definition",
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
],
|
|
71
|
+
term: capsule.term,
|
|
72
|
+
meaning: capsule.meaning,
|
|
73
|
+
evidence: capsule.evidence.map((evidence) => ({
|
|
74
|
+
source_id: String(evidence.source_id),
|
|
75
|
+
evidence_anchor: String(evidence.evidence_anchor),
|
|
76
|
+
signal_type: String(evidence.signal_type),
|
|
77
|
+
})),
|
|
104
78
|
});
|
|
105
79
|
}
|
|
106
80
|
for (const term of new Set(input.requestedTerms.map((value) => value.trim()).filter(Boolean))) {
|
|
107
81
|
const evidence = [...resolved.values()]
|
|
108
82
|
.filter(({ signal, record }) => inferredTypes.has(signal.signal_type) &&
|
|
109
83
|
inferredKinds.has(signal.source_kind) &&
|
|
84
|
+
hasInferredProvenance(signal) &&
|
|
110
85
|
semanticValues(record).some((value) => containsCompleteTerm(value, term)))
|
|
111
86
|
.map(({ signal }) => ({
|
|
112
87
|
source_id: signal.source_id,
|
|
@@ -116,10 +91,10 @@ export function admitPersonalGlossaryEvidence(input, resolveAnchor = resolveEvid
|
|
|
116
91
|
.filter((item, index, all) => all.findIndex((candidate) => candidate.source_id === item.source_id ||
|
|
117
92
|
candidate.evidence_anchor === item.evidence_anchor) === index)
|
|
118
93
|
.sort((left, right) => left.source_id.localeCompare(right.source_id));
|
|
119
|
-
if (evidence.length
|
|
120
|
-
new Set(evidence.map((item) => item.source_id)).size
|
|
121
|
-
new Set(evidence.map((item) => item.evidence_anchor)).size
|
|
122
|
-
candidates.push({ kind: "personal_inferred_usage", term, evidence
|
|
94
|
+
if (evidence.length === 2 &&
|
|
95
|
+
new Set(evidence.map((item) => item.source_id)).size === 2 &&
|
|
96
|
+
new Set(evidence.map((item) => item.evidence_anchor)).size === 2) {
|
|
97
|
+
candidates.push({ kind: "personal_inferred_usage", term, evidence });
|
|
123
98
|
}
|
|
124
99
|
}
|
|
125
100
|
return candidates.length > 0
|
|
@@ -131,3 +106,8 @@ export function admitPersonalGlossaryEvidence(input, resolveAnchor = resolveEvid
|
|
|
131
106
|
recovery: authority.insufficientRecovery,
|
|
132
107
|
};
|
|
133
108
|
}
|
|
109
|
+
export { classifyExplicitGlossaryLanguage, discoverExplicitGlossaryCues, mineExplicitGlossaryCandidates, mineExplicitGlossaryEvidence, minePersonalExplicitGlossaryCandidates, EXPLICIT_GLOSSARY_REASONS, } from "./personalGlossaryExplicit.js";
|
|
110
|
+
export { mineRecurringGlossaryCandidates, mineRecurringGlossaryEvidence, PERSONAL_GLOSSARY_MINING_POLICY_VERSION, RECURRING_MEANING_PENDING_REVIEW, RECURRING_REASONS, } from "./personalGlossaryRecurrence.js";
|
|
111
|
+
export { maintainPersonalGlossaryCandidateProjection, personalGlossaryCandidateProjectionPath, personalGlossaryProjectionProjectIdentity, persistPersonalGlossaryCandidateProjection, projectPersonalGlossaryCandidates, readPersonalGlossaryCandidateProjection, } from "./personalGlossaryCandidateProjection.js";
|
|
112
|
+
export { decidePersonalGlossaryCandidate, } from "./personalGlossaryDecision.js";
|
|
113
|
+
export { currentPersonalGlossaryReviewRecords, maintainPersonalGlossaryReviewRecords, personalGlossaryReviewRecordsPath, queuePersonalGlossaryReviewRecord, readPersonalGlossaryReviewRecords, } from "./personalGlossaryReviewRecords.js";
|