ds4-context-engine 0.1.2 → 0.2.0-beta.2

package/quality/corpus-v1.json

@@ -0,0 +1,208 @@
+{
+  "schemaVersion": 1,
+  "corpusVersion": "ds4-quality-corpus-v1",
+  "description": "Synthetic metadata-only replay fixtures for deterministic DS4 context-quality comparisons.",
+  "fixtures": [
+    {
+      "schemaVersion": 1,
+      "corpusVersion": "ds4-quality-corpus-v1",
+      "sampleId": "fixture:retrieval-budget",
+      "plannerVersion": "managed-v1",
+      "profileKey": "fixture/model-128k",
+      "task": {
+        "taskId": "task:retrieval-budget",
+        "taskType": "debugging",
+        "queryTags": ["exact-symbol", "historical-decision"]
+      },
+      "expectedEvidenceSourceIds": ["entry:decision-a"],
+      "candidates": [
+        {
+          "id": "candidate:current-a",
+          "kind": "current",
+          "tokens": 80,
+          "groupId": "group:current-a",
+          "sourceIds": ["entry:request-a"],
+          "staticScore": 1000,
+          "candidateScore": 1000,
+          "currentRequest": true
+        },
+        {
+          "id": "candidate:recent-a",
+          "kind": "recent",
+          "tokens": 120,
+          "groupId": "group:recent-a",
+          "sourceIds": ["entry:recent-a"],
+          "staticScore": 100,
+          "candidateScore": 40
+        },
+        {
+          "id": "candidate:retrieval-a",
+          "kind": "retrieval",
+          "tokens": 100,
+          "groupId": "group:retrieval-a",
+          "sourceIds": ["entry:decision-a"],
+          "staticScore": 85,
+          "candidateScore": 180
+        }
+      ],
+      "totalBudgetTokens": 210,
+      "categoryBudgets": {
+        "recent": 160,
+        "retrieval": 120
+      },
+      "outcomeLabels": ["expected:decision-recovered"]
+    },
+    {
+      "schemaVersion": 1,
+      "corpusVersion": "ds4-quality-corpus-v1",
+      "sampleId": "fixture:atomic-tools",
+      "plannerVersion": "managed-v1",
+      "profileKey": "fixture/model-32k",
+      "task": {
+        "taskId": "task:atomic-tools",
+        "taskType": "tool-debugging",
+        "queryTags": ["atomic-tool-group"]
+      },
+      "expectedEvidenceSourceIds": ["entry:tool-call", "entry:tool-result"],
+      "candidates": [
+        {
+          "id": "candidate:current-b",
+          "kind": "current",
+          "tokens": 40,
+          "groupId": "group:current-b",
+          "sourceIds": ["entry:request-b"],
+          "staticScore": 1000,
+          "candidateScore": 1000,
+          "currentRequest": true
+        },
+        {
+          "id": "candidate:tool-call",
+          "kind": "retrieval",
+          "tokens": 60,
+          "groupId": "group:tool-exchange",
+          "sourceIds": ["entry:tool-call"],
+          "staticScore": 90,
+          "candidateScore": 120
+        },
+        {
+          "id": "candidate:tool-result",
+          "kind": "retrieval",
+          "tokens": 100,
+          "groupId": "group:tool-exchange",
+          "sourceIds": ["entry:tool-result"],
+          "staticScore": 90,
+          "candidateScore": 120
+        },
+        {
+          "id": "candidate:background-b",
+          "kind": "recent",
+          "tokens": 170,
+          "groupId": "group:background-b",
+          "sourceIds": ["entry:background-b"],
+          "staticScore": 80,
+          "candidateScore": 20
+        }
+      ],
+      "totalBudgetTokens": 220,
+      "categoryBudgets": {
+        "recent": 180,
+        "retrieval": 180
+      },
+      "outcomeLabels": ["expected:atomic-selection"]
+    },
+    {
+      "schemaVersion": 1,
+      "corpusVersion": "ds4-quality-corpus-v1",
+      "sampleId": "fixture:duplicate-evidence",
+      "plannerVersion": "managed-v1",
+      "profileKey": "fixture/model-200k",
+      "task": {
+        "taskId": "task:duplicate-evidence",
+        "taskType": "refactoring",
+        "queryTags": ["decision-recall", "duplicate-control"]
+      },
+      "expectedEvidenceSourceIds": ["entry:decision-b", "entry:decision-c"],
+      "candidates": [
+        {
+          "id": "candidate:current-c",
+          "kind": "current",
+          "tokens": 50,
+          "groupId": "group:current-c",
+          "sourceIds": ["entry:request-c"],
+          "staticScore": 1000,
+          "candidateScore": 1000,
+          "currentRequest": true
+        },
+        {
+          "id": "candidate:decision-b-primary",
+          "kind": "retrieval",
+          "tokens": 80,
+          "groupId": "group:decision-b-primary",
+          "sourceIds": ["entry:decision-b"],
+          "staticScore": 100,
+          "candidateScore": 100
+        },
+        {
+          "id": "candidate:decision-b-duplicate",
+          "kind": "retrieval",
+          "tokens": 80,
+          "groupId": "group:decision-b-duplicate",
+          "sourceIds": ["entry:decision-b"],
+          "staticScore": 99,
+          "candidateScore": 10
+        },
+        {
+          "id": "candidate:decision-c",
+          "kind": "retrieval",
+          "tokens": 80,
+          "groupId": "group:decision-c",
+          "sourceIds": ["entry:decision-c"],
+          "staticScore": 98,
+          "candidateScore": 99
+        }
+      ],
+      "totalBudgetTokens": 210,
+      "categoryBudgets": {
+        "retrieval": 160
+      },
+      "outcomeLabels": ["expected:no-duplicate"]
+    },
+    {
+      "schemaVersion": 1,
+      "corpusVersion": "ds4-quality-corpus-v1",
+      "sampleId": "fixture:mandatory-overflow",
+      "plannerVersion": "managed-v1",
+      "profileKey": "fixture/model-32k",
+      "task": {
+        "taskId": "task:mandatory-overflow",
+        "taskType": "overflow-safety",
+        "queryTags": ["fail-open", "current-request"]
+      },
+      "expectedEvidenceSourceIds": [],
+      "candidates": [
+        {
+          "id": "candidate:current-d",
+          "kind": "current",
+          "tokens": 300,
+          "groupId": "group:current-d",
+          "sourceIds": ["entry:request-d"],
+          "staticScore": 1000,
+          "candidateScore": 1000,
+          "currentRequest": true
+        },
+        {
+          "id": "candidate:history-d",
+          "kind": "history",
+          "tokens": 50,
+          "groupId": "group:history-d",
+          "sourceIds": ["entry:history-d"],
+          "staticScore": 10,
+          "candidateScore": 10
+        }
+      ],
+      "totalBudgetTokens": 200,
+      "categoryBudgets": {},
+      "outcomeLabels": ["expected:fail-open"]
+    }
+  ]
+}

package/quality/semantic-corpus-v1.json

@@ -0,0 +1,84 @@
+{
+  "schemaVersion": 1,
+  "corpusVersion": "ds4-semantic-corpus-v1",
+  "baseQualityCorpusVersion": "ds4-quality-corpus-v1",
+  "description": "Synthetic sanitized M16 retrieval fixtures evaluated with the M14 evidence recall and irrelevant-token metrics.",
+  "fixtures": [
+    {
+      "id": "semantic-auth-renewal",
+      "query": "renew authorization credentials after expiry",
+      "exactIdentifiers": [],
+      "expectedEvidenceIds": ["evidence:auth-refresh"],
+      "maxResults": 1,
+      "candidates": [
+        {
+          "id": "evidence:auth-refresh",
+          "text": "OAuth token refresh handles expired authentication secrets.",
+          "tokens": 24
+        },
+        {
+          "id": "evidence:schema",
+          "text": "SQLite migration creates a project metadata table.",
+          "tokens": 28
+        }
+      ]
+    },
+    {
+      "id": "semantic-cache-reuse",
+      "query": "reuse previously computed responses to reduce latency",
+      "exactIdentifiers": [],
+      "expectedEvidenceIds": ["evidence:cache"],
+      "maxResults": 1,
+      "candidates": [
+        {
+          "id": "evidence:cache",
+          "text": "Memoized output caching avoids repeated work and improves performance.",
+          "tokens": 30
+        },
+        {
+          "id": "evidence:logging",
+          "text": "Structured logs record request metadata for diagnostics.",
+          "tokens": 26
+        }
+      ]
+    },
+    {
+      "id": "exact-path-priority",
+      "query": "inspect src/Auth.ts before changing token renewal",
+      "exactIdentifiers": ["src/Auth.ts"],
+      "expectedEvidenceIds": ["evidence:exact-path"],
+      "maxResults": 1,
+      "candidates": [
+        {
+          "id": "evidence:exact-path",
+          "text": "src/Auth.ts exports a small compatibility wrapper.",
+          "tokens": 20
+        },
+        {
+          "id": "evidence:semantic-neighbor",
+          "text": "Authentication credential refresh renews expired OAuth tokens.",
+          "tokens": 34
+        }
+      ]
+    },
+    {
+      "id": "semantic-storage-failure",
+      "query": "debug persistence failures in the local datastore",
+      "exactIdentifiers": [],
+      "expectedEvidenceIds": ["evidence:sqlite-error"],
+      "maxResults": 1,
+      "candidates": [
+        {
+          "id": "evidence:sqlite-error",
+          "text": "SQLite database exceptions are isolated and fall back safely.",
+          "tokens": 32
+        },
+        {
+          "id": "evidence:ui",
+          "text": "Terminal colors and status labels are rendered by the UI.",
+          "tokens": 30
+        }
+      ]
+    }
+  ]
+}

package/quality/symbol-corpus-v1.json

@@ -0,0 +1,38 @@
+{
+  "schemaVersion": 1,
+  "corpusVersion": "ds4-symbol-corpus-v1",
+  "fixtures": [
+    {
+      "id": "typescript-comments-and-methods",
+      "language": "typescript",
+      "filePath": "src/service.ts",
+      "content": "import { Client } from \"./client.js\";\n// class PhantomComment {}\nconst sample = \"function PhantomString() {}\";\nexport class Service {\n  async run(client: Client): Promise<Result> {\n    return client.execute();\n  }\n}\n",
+      "expectedSymbols": ["sample", "Service", "Service.run"],
+      "forbiddenSymbols": ["PhantomComment", "PhantomString"]
+    },
+    {
+      "id": "javascript-declarations",
+      "language": "javascript",
+      "filePath": "src/worker.js",
+      "content": "const text = 'class StringOnly {}';\nexport function buildWorker(options) {\n  return createWorker(options);\n}\nexport const stopWorker = () => terminateWorker();\n",
+      "expectedSymbols": ["text", "buildWorker", "stopWorker"],
+      "forbiddenSymbols": ["StringOnly"]
+    },
+    {
+      "id": "python-indentation",
+      "language": "python",
+      "filePath": "src/processor.py",
+      "content": "from app.client import Client\n# def phantom_comment(): pass\nclass Processor:\n    def process(self, client: Client):\n        return client.execute()\n\ndef create_processor():\n    return Processor()\n",
+      "expectedSymbols": ["Processor", "Processor.process", "create_processor"],
+      "forbiddenSymbols": ["phantom_comment"]
+    },
+    {
+      "id": "go-receiver-method",
+      "language": "go",
+      "filePath": "service/service.go",
+      "content": "package service\n\nimport (\n  \"context\"\n  \"example/client\"\n)\n\n// type Phantom struct{}\ntype Service struct { client *client.Client }\n\nfunc (s *Service) Run(ctx context.Context) error {\n  return s.client.Execute(ctx)\n}\n",
+      "expectedSymbols": ["Service", "Service.Run"],
+      "forbiddenSymbols": ["Phantom"]
+    }
+  ]
+}

package/scripts/compare-context-quality.mjs

@@ -0,0 +1,23 @@
+import { readFileSync } from "node:fs";
+import { performance } from "node:perf_hooks";
+import { fileURLToPath } from "node:url";
+import {
+  compareQualityStrategies,
+  staticRankingStrategy,
+  taskWeightedCandidateStrategy,
+} from "ds4-context-core/quality/context-quality";
+import { stableStringify } from "ds4-context-core/shared/stable-json";
+
+const corpusUrl = new URL("../quality/corpus-v1.json", import.meta.url);
+const corpus = JSON.parse(readFileSync(fileURLToPath(corpusUrl), "utf8"));
+const startedAt = performance.now();
+const report = compareQualityStrategies(corpus, [
+  staticRankingStrategy(),
+  taskWeightedCandidateStrategy(),
+]);
+const durationMs = performance.now() - startedAt;
+
+process.stdout.write(`${stableStringify(report)}\n`);
+if (process.argv.includes("--timing")) {
+  process.stderr.write(`quality comparison wall-clock: ${durationMs.toFixed(3)} ms\n`);
+}