nlm-memory 0.20.0 → 0.20.1

package/fixtures/classifier-gold/reference.json

@@ -0,0 +1,318 @@
+[
+  {
+    "id": "01-bug-fix-null-pointer-api",
+    "label": "Fix null pointer bug in corvid-api events route",
+    "labelAlternates": [
+      "corvid-api POST events null source fix",
+      "Fix missing source field TypeError in events handler"
+    ],
+    "entities": [
+      "corvid-api",
+      "Express",
+      "Postgres"
+    ],
+    "decisions": [
+      "Reject requests missing source field with 400 instead of defaulting"
+    ],
+    "expectLowConfidence": false
+  },
+  {
+    "id": "02-bug-fix-race-condition-queue",
+    "label": "Fix duplicate job execution race condition in taskvine",
+    "labelAlternates": [
+      "taskvine worker race condition fix with SELECT FOR UPDATE",
+      "Fix concurrent worker job claim race in taskvine"
+    ],
+    "entities": [
+      "taskvine",
+      "PostgreSQL",
+      "SQLAlchemy"
+    ],
+    "decisions": [
+      "Replace unlocked SELECT with SELECT FOR UPDATE SKIP LOCKED to eliminate duplicate job claims"
+    ],
+    "expectLowConfidence": false
+  },
+  {
+    "id": "03-bug-fix-jwt-clock-skew",
+    "label": "Fix JWT clock skew validation failures in bridgeauth",
+    "labelAlternates": [
+      "bridgeauth JWT expiry leeway fix",
+      "Add clock skew leeway to PyJWT token validation"
+    ],
+    "entities": [
+      "bridgeauth",
+      "PyJWT",
+      "HS256"
+    ],
+    "decisions": [
+      "Add clock-skew leeway to JWT validation"
+    ],
+    "expectLowConfidence": false
+  },
+  {
+    "id": "04-feature-webhook-retry",
+    "label": "Add exponential backoff webhook retry queue to taskstream",
+    "labelAlternates": [
+      "taskstream webhook retry with Redis sorted set",
+      "Implement webhook retry with exponential backoff in taskstream"
+    ],
+    "entities": [
+      "taskstream",
+      "Redis"
+    ],
+    "decisions": [
+      "Use Redis sorted set with score as next-attempt timestamp for retry queue",
+      "Dead-letter failed jobs after 5 attempts with exponential backoff"
+    ],
+    "expectLowConfidence": false
+  },
+  {
+    "id": "05-feature-csv-export",
+    "label": "Add streaming CSV export endpoint to lumens",
+    "labelAlternates": [
+      "lumens GET /reports/export CSV streaming",
+      "Implement CSV export with streaming response in lumens"
+    ],
+    "entities": [
+      "lumens",
+      "encoding/csv"
+    ],
+    "decisions": [
+      "Stream CSV directly to http.ResponseWriter instead of buffering in memory"
+    ],
+    "expectLowConfidence": false
+  },
+  {
+    "id": "06-feature-rate-limiting",
+    "label": "Add Redis-backed rate limiting to crestapi search endpoint",
+    "labelAlternates": [
+      "crestapi API rate limiting with slowapi and Redis",
+      "Implement per-API-key rate limiting in crestapi"
+    ],
+    "entities": [
+      "crestapi",
+      "FastAPI",
+      "Redis",
+      "slowapi"
+    ],
+    "decisions": [
+      "Use slowapi with Redis backend for 100 requests per minute per API key"
+    ],
+    "expectLowConfidence": false
+  },
+  {
+    "id": "07-refactor-db-abstraction",
+    "label": "Extract DbAdapter interface to ports layer in vaultdb",
+    "labelAlternates": [
+      "vaultdb database adapter abstraction refactor",
+      "Refactor vaultdb to depend on DbAdapter interface not Postgres directly"
+    ],
+    "entities": [
+      "vaultdb",
+      "TypeScript",
+      "Postgres",
+      "SQLite"
+    ],
+    "decisions": [
+      "Extract DbAdapter interface to ports layer so core modules do not import Postgres directly"
+    ],
+    "expectLowConfidence": false
+  },
+  {
+    "id": "08-refactor-config-loading",
+    "label": "Centralize config into Pydantic Settings class in axiom-worker",
+    "labelAlternates": [
+      "axiom-worker config refactor using Pydantic Settings",
+      "Replace scattered os.getenv calls with Pydantic Settings in axiom-worker"
+    ],
+    "entities": [
+      "axiom-worker",
+      "Pydantic",
+      "pydantic-settings"
+    ],
+    "decisions": [
+      "Replace scattered os.getenv calls with a single Pydantic Settings class loaded at startup"
+    ],
+    "expectLowConfidence": false
+  },
+  {
+    "id": "09-ops-docker-staging",
+    "label": "Set up Docker Compose staging environment with Traefik for droneflow",
+    "labelAlternates": [
+      "droneflow staging docker-compose with Traefik and TLS",
+      "Create staging Docker Compose stack for droneflow"
+    ],
+    "entities": [
+      "droneflow",
+      "Docker Compose",
+      "Traefik"
+    ],
+    "decisions": [
+      "Use Traefik for TLS termination and routing in staging",
+      "Keep staging deploys explicit via deploy script rather than auto-pull"
+    ],
+    "expectLowConfidence": false
+  },
+  {
+    "id": "10-ops-nginx-ssl",
+    "label": "Configure nginx reverse proxy with TLS for gridhost",
+    "labelAlternates": [
+      "gridhost nginx TLS setup with certbot and Let's Encrypt",
+      "Set up nginx with Let's Encrypt certificate for gridhost"
+    ],
+    "entities": [
+      "gridhost",
+      "nginx",
+      "certbot"
+    ],
+    "decisions": [
+      "Terminate TLS at nginx and proxy to app on port 8080",
+      "Add cron fallback for certbot renewal in case systemd timer is disabled"
+    ],
+    "expectLowConfidence": false
+  },
+  {
+    "id": "11-ops-postgres-migration",
+    "label": "Run concurrent index migration on Postgres sessions table",
+    "labelAlternates": [
+      "Add CONCURRENTLY indexes to sessions table via Alembic",
+      "Postgres sessions table index migration without table lock"
+    ],
+    "entities": [
+      "PostgreSQL",
+      "Alembic"
+    ],
+    "decisions": [
+      "Use CREATE INDEX CONCURRENTLY to avoid locking the 8-million-row sessions table",
+      "Set statement_timeout to 0 for the migration connection to prevent timeout killing the index build"
+    ],
+    "expectLowConfidence": false
+  },
+  {
+    "id": "12-ops-cron-debugging",
+    "label": "Debug and fix flarepath nightly cron job not running",
+    "labelAlternates": [
+      "flarepath cron exit 126 permission denied fix",
+      "Fix missing execute bit on flarepath nightly report script"
+    ],
+    "entities": [
+      "flarepath"
+    ],
+    "decisions": [
+      "Add chmod to deploy script to re-apply execute bit after git checkout"
+    ],
+    "expectLowConfidence": false
+  },
+  {
+    "id": "13-research-vector-db-comparison",
+    "label": "Vector database comparison and Qdrant selection for pulsearch",
+    "labelAlternates": [
+      "pulsearch vector DB evaluation: pgvector vs Qdrant vs Weaviate",
+      "Choose Qdrant for pulsearch vector search"
+    ],
+    "entities": [
+      "pulsearch",
+      "Qdrant",
+      "pgvector",
+      "Weaviate"
+    ],
+    "decisions": [
+      "Use Qdrant for pulsearch instead of pgvector due to stronger ANN performance and pre-filter payload filtering"
+    ],
+    "expectLowConfidence": false
+  },
+  {
+    "id": "14-research-oauth-provider",
+    "label": "OAuth provider evaluation and Keycloak selection for irongate",
+    "labelAlternates": [
+      "irongate auth provider comparison: Keycloak vs Auth0 vs SuperTokens",
+      "Choose Keycloak self-hosted for irongate customer auth"
+    ],
+    "entities": [
+      "irongate",
+      "Keycloak",
+      "Auth0",
+      "SuperTokens"
+    ],
+    "decisions": [
+      "Use Keycloak self-hosted backed by Postgres for irongate auth because it covers LDAP federation without additional cost"
+    ],
+    "expectLowConfidence": false
+  },
+  {
+    "id": "15-writing-api-docs",
+    "label": "Draft OpenAPI 3.0 documentation for cobalt-api with Swagger UI",
+    "labelAlternates": [
+      "cobalt-api OpenAPI spec and Swagger UI setup",
+      "Set up OpenAPI docs for cobalt-api served at /api-docs"
+    ],
+    "entities": [
+      "cobalt-api",
+      "OpenAPI",
+      "Swagger UI"
+    ],
+    "decisions": [
+      "Use OpenAPI 3.0 format and host docs publicly at /api-docs via swagger-ui-express"
+    ],
+    "expectLowConfidence": false
+  },
+  {
+    "id": "16-meeting-sprint-planning",
+    "label": "Sprint 5 planning notes for orbitflow team",
+    "labelAlternates": [
+      "orbitflow sprint 5 planning: webhook retry, rate limiting, deferrals",
+      "orbitflow sprint 5 commitments and deferrals"
+    ],
+    "entities": [
+      "orbitflow"
+    ],
+    "decisions": [
+      "Webhook retry is top backend priority for sprint 5",
+      "Analytics dashboard deferred to sprint 6",
+      "Rate limiting assigned to backend as secondary sprint 5 item",
+      "Mobile push notifications deferred to sprint 6 backlog"
+    ],
+    "expectLowConfidence": false
+  },
+  {
+    "id": "17-trivial-list-files",
+    "label": "List files in src directory",
+    "labelAlternates": [
+      "Directory listing of src folder"
+    ],
+    "entities": [],
+    "decisions": [],
+    "expectLowConfidence": true
+  },
+  {
+    "id": "18-trivial-version-check",
+    "label": "Check Node.js and npm versions",
+    "labelAlternates": [
+      "Node.js npm version query"
+    ],
+    "entities": [],
+    "decisions": [],
+    "expectLowConfidence": true
+  },
+  {
+    "id": "19-trivial-syntax-help",
+    "label": "TypeScript array iteration syntax question",
+    "labelAlternates": [
+      "How to iterate array with index in TypeScript"
+    ],
+    "entities": [],
+    "decisions": [],
+    "expectLowConfidence": true
+  },
+  {
+    "id": "20-trivial-read-package-json",
+    "label": "Read package.json and confirm dependencies installed",
+    "labelAlternates": [
+      "Check installed npm dependencies from package.json"
+    ],
+    "entities": [],
+    "decisions": [],
+    "expectLowConfidence": true
+  }
+]

package/fixtures/classifier-gold/transcripts/01-bug-fix-null-pointer-api.txt

@@ -0,0 +1,75 @@
+User: The POST /events endpoint in corvid-api is throwing a TypeError in production. Stack trace says "Cannot read properties of undefined (reading 'source')" on line 47 of src/routes/events.ts. Can you find the bug?
+
+Agent: Let me read the route handler.
+
+[Read: src/routes/events.ts]
+
+```typescript
+export async function postEvent(req: Request, res: Response) {
+  const { source, payload } = req.body;
+  // line 47
+  const normalized = source.toLowerCase();
+  await db.events.insert({ source: normalized, payload, createdAt: new Date() });
+  res.status(201).json({ ok: true });
+}
+```
+
+The bug is on line 47. When the request body omits the `source` field, `source` is `undefined` and calling `.toLowerCase()` throws. There is no guard before the property access.
+
+User: Right. What's the fix?
+
+Agent: Two options: reject requests missing `source` with a 400, or default it to "unknown". Given this is an event ingestion endpoint where source identity matters for routing downstream, rejecting is safer.
+
+User: Agreed. Reject with 400.
+
+Agent: Updating the handler now.
+
+[Edit: src/routes/events.ts]
+
+```typescript
+export async function postEvent(req: Request, res: Response) {
+  const { source, payload } = req.body;
+  if (typeof source !== "string" || source.trim().length === 0) {
+    res.status(400).json({ error: "source is required" });
+    return;
+  }
+  const normalized = source.toLowerCase().trim();
+  await db.events.insert({ source: normalized, payload, createdAt: new Date() });
+  res.status(201).json({ ok: true });
+}
+```
+
+User: Good. Add a test for the missing-source case.
+
+Agent: Adding to tests/routes/events.test.ts.
+
+[Read: tests/routes/events.test.ts]
+
+[Edit: tests/routes/events.test.ts]
+
+Added a test case:
+
+```typescript
+it("returns 400 when source is missing", async () => {
+  const res = await request(app).post("/events").send({ payload: { x: 1 } });
+  expect(res.status).toBe(400);
+  expect(res.body.error).toBe("source is required");
+});
+```
+
+User: Run the tests to confirm.
+
+Agent: [Bash: npm test -- --testPathPattern=events]
+
+Output:
+```
+PASS tests/routes/events.test.ts
+  POST /events
+    ✓ returns 201 for valid event (42ms)
+    ✓ returns 400 when source is missing (8ms)
+    ✓ returns 400 when source is empty string (6ms)
+```
+
+All passing. The fix guards on both undefined and empty-string cases since `trim().length === 0` catches whitespace-only inputs too.
+
+User: Perfect. Ship it.

package/fixtures/classifier-gold/transcripts/02-bug-fix-race-condition-queue.txt

@@ -0,0 +1,77 @@
+User: We're seeing duplicate job execution in taskvine. Two workers occasionally pick up the same job from the queue table. The claim logic is in src/worker/claim.py.
+
+Agent: [Read: src/worker/claim.py]
+
+```python
+def claim_next_job(conn):
+    with conn.transaction():
+        row = conn.execute(
+            "SELECT id FROM jobs WHERE status = 'pending' LIMIT 1"
+        ).fetchone()
+        if row is None:
+            return None
+        conn.execute(
+            "UPDATE jobs SET status = 'running', worker_id = %s WHERE id = %s",
+            (WORKER_ID, row["id"]),
+        )
+        return row["id"]
+```
+
+This is a classic race condition. The SELECT and UPDATE are in the same transaction, but between the SELECT and UPDATE another worker can read the same row before either has committed. The advisory lock approach you're using doesn't appear here, so there's no row-level locking.
+
+The fix is to use SELECT FOR UPDATE SKIP LOCKED. This atomically locks the selected row for the duration of the transaction and skips rows already locked by another worker.
+
+User: Yes, let's switch to SELECT FOR UPDATE SKIP LOCKED.
+
+Agent: [Edit: src/worker/claim.py]
+
+```python
+def claim_next_job(conn):
+    with conn.transaction():
+        row = conn.execute(
+            """
+            SELECT id FROM jobs
+            WHERE status = 'pending'
+            ORDER BY created_at
+            LIMIT 1
+            FOR UPDATE SKIP LOCKED
+            """
+        ).fetchone()
+        if row is None:
+            return None
+        conn.execute(
+            "UPDATE jobs SET status = 'running', worker_id = %s WHERE id = %s",
+            (WORKER_ID, row["id"]),
+        )
+        return row["id"]
+```
+
+Added ORDER BY created_at for FIFO fairness while I was in there.
+
+User: Does PostgreSQL actually guarantee no duplicates with this pattern?
+
+Agent: Yes. FOR UPDATE acquires a row-level lock at SELECT time. SKIP LOCKED means any row already locked by another transaction is skipped entirely rather than waited on. Within a single transaction, any row your SELECT returns is guaranteed not to be selected by another concurrent transaction using the same pattern. This is the canonical pattern for Postgres-backed job queues.
+
+User: What about the existing advisory lock code I thought we had?
+
+Agent: [Bash: grep -r "advisory\|pg_try_advisory" src/]
+
+No results. There is no advisory lock code in the codebase. The previous SELECT without FOR UPDATE had no locking at all, which is why duplicates occurred.
+
+User: Got it. Let's also add a test to confirm two concurrent workers cannot claim the same job.
+
+Agent: [Read: tests/worker/test_claim.py]
+
+[Edit: tests/worker/test_claim.py]
+
+Added a test that spawns two threads, each calling claim_next_job with a shared connection pool containing a single pending job, and asserts only one gets a non-None return and the other gets None.
+
+User: Run it.
+
+Agent: [Bash: pytest tests/worker/test_claim.py -v]
+
+```
+tests/worker/test_claim.py::test_concurrent_claim_no_duplicate PASSED
+```
+
+Fixed. The SELECT FOR UPDATE SKIP LOCKED pattern eliminates the race.

package/fixtures/classifier-gold/transcripts/03-bug-fix-jwt-clock-skew.txt

@@ -0,0 +1,86 @@
+User: Users are reporting random login failures in bridgeauth. The error in logs is "Token is not yet valid" and "Signature has expired". It's intermittent and seems worse around the token boundary. Here's the validation code in src/auth/token.py.
+
+Agent: [Read: src/auth/token.py]
+
+```python
+import jwt
+
+SECRET = os.environ["JWT_SECRET"]
+ALGORITHM = "HS256"
+
+def validate_token(token: str) -> dict:
+    return jwt.decode(token, SECRET, algorithms=[ALGORITHM])
+```
+
+The validation call has no leeway parameter. JWT tokens have an `exp` claim and PyJWT validates it against the current time. When there is a small clock difference between the server that issued the token and the server validating it, tokens can appear expired seconds before their actual expiry from the issuer's perspective.
+
+The standard fix is to add a leeway value, typically 10-30 seconds, which tells PyJWT to accept tokens that expired within that window.
+
+User: How big is the clock drift between our services likely to be?
+
+Agent: For services within the same datacenter using NTP, drift is usually under 1 second. But network-synchronized clocks under load can drift up to 5 seconds. A 10-second leeway covers realistic cases without meaningfully extending token validity from a security perspective.
+
+User: Let's add 10 seconds of leeway.
+
+Agent: [Edit: src/auth/token.py]
+
+```python
+import jwt
+from datetime import timedelta
+
+SECRET = os.environ["JWT_SECRET"]
+ALGORITHM = "HS256"
+CLOCK_SKEW_LEEWAY = timedelta(seconds=10)
+
+def validate_token(token: str) -> dict:
+    return jwt.decode(
+        token,
+        SECRET,
+        algorithms=[ALGORITHM],
+        leeway=CLOCK_SKEW_LEEWAY,
+    )
+```
+
+User: Are there tests for the expiry behavior?
+
+Agent: [Bash: grep -n "expir\|leeway\|exp" tests/auth/test_token.py]
+
+```
+12: def make_token(exp_offset_seconds=300):
+14:     payload = {"sub": "user-1", "exp": now + timedelta(seconds=exp_offset_seconds)}
+```
+
+No test covers the boundary case. I'll add one that creates a token with `exp` set 5 seconds in the past and confirms it is now accepted with leeway.
+
+[Edit: tests/auth/test_token.py]
+
+Added:
+
+```python
+def test_validates_token_within_leeway():
+    # Token expired 5 seconds ago; should pass with 10s leeway
+    payload = {"sub": "user-1", "exp": datetime.utcnow() - timedelta(seconds=5)}
+    token = jwt.encode(payload, SECRET, algorithm="HS256")
+    result = validate_token(token)
+    assert result["sub"] == "user-1"
+
+def test_rejects_token_outside_leeway():
+    # Token expired 60 seconds ago; must fail even with leeway
+    payload = {"sub": "user-1", "exp": datetime.utcnow() - timedelta(seconds=60)}
+    token = jwt.encode(payload, SECRET, algorithm="HS256")
+    with pytest.raises(jwt.ExpiredSignatureError):
+        validate_token(token)
+```
+
+User: Run the tests.
+
+Agent: [Bash: pytest tests/auth/test_token.py -v]
+
+```
+tests/auth/test_token.py::test_validates_valid_token PASSED
+tests/auth/test_token.py::test_rejects_expired_token PASSED
+tests/auth/test_token.py::test_validates_token_within_leeway PASSED
+tests/auth/test_token.py::test_rejects_token_outside_leeway PASSED
+```
+
+All green. The 10-second leeway resolves the clock-skew login failures.

package/fixtures/classifier-gold/transcripts/04-feature-webhook-retry.txt

@@ -0,0 +1,95 @@
+User: taskstream needs exponential backoff retry for failed webhook deliveries. Right now if a delivery fails it just drops the event. We need up to 5 retry attempts with exponential backoff, delays stored in Redis so retries survive restarts.
+
+Agent: I'll design this before writing code. The approach: use a Redis sorted set where the score is the Unix timestamp of the next attempt. A background worker polls for jobs whose score is <= now, attempts delivery, and either marks the job done or reschedules it with a doubled delay. After 5 attempts the job is moved to a dead-letter key.
+
+Does this fit the existing architecture? Let me check what's already in place.
+
+[Read: src/delivery/deliver.ts]
+[Read: src/worker/retry-worker.ts]
+
+The deliver.ts module handles HTTP POST; retry-worker.ts exists but is a stub. Good starting point.
+
+User: That architecture sounds right. Go ahead.
+
+Agent: [Edit: src/delivery/retry-queue.ts] (new file)
+
+```typescript
+import { createClient } from "redis";
+
+const RETRY_KEY = "taskstream:retry";
+const DEAD_KEY  = "taskstream:dead";
+const MAX_ATTEMPTS = 5;
+const BASE_DELAY_MS = 5_000;
+
+export async function enqueueRetry(
+  redis: ReturnType<typeof createClient>,
+  jobId: string,
+  attempt: number,
+): Promise<void> {
+  if (attempt >= MAX_ATTEMPTS) {
+    await redis.lPush(DEAD_KEY, jobId);
+    return;
+  }
+  const delayMs = BASE_DELAY_MS * Math.pow(2, attempt);
+  const score = Date.now() + delayMs;
+  await redis.zAdd(RETRY_KEY, { score, value: jobId });
+}
+
+export async function pollDueRetries(
+  redis: ReturnType<typeof createClient>,
+): Promise<string[]> {
+  const now = Date.now();
+  return redis.zRangeByScore(RETRY_KEY, "-inf", now);
+}
+
+export async function removeFromQueue(
+  redis: ReturnType<typeof createClient>,
+  jobId: string,
+): Promise<void> {
+  await redis.zRem(RETRY_KEY, jobId);
+}
+```
+
+[Edit: src/worker/retry-worker.ts]
+
+Updated the stub to poll every 5 seconds, attempt delivery for each due job, and call enqueueRetry with incremented attempt count on failure or removeFromQueue on success.
+
+User: What's the delay schedule for 5 attempts?
+
+Agent: With BASE_DELAY_MS = 5000 and doubling per attempt:
+- Attempt 0 -> 1: 5 seconds
+- Attempt 1 -> 2: 10 seconds
+- Attempt 2 -> 3: 20 seconds
+- Attempt 3 -> 4: 40 seconds
+- Attempt 4 -> 5: dead letter (no 6th attempt)
+
+Total window before dead-letter: ~75 seconds. If you want longer, we can increase BASE_DELAY_MS or switch from 2x to a larger multiplier.
+
+User: 75 seconds is fine for v1. What key format is used for job IDs?
+
+Agent: [Read: src/jobs/job-id.ts]
+
+Job IDs are UUIDs generated at event creation time, e.g. `evt_a3f7c290`. The retry queue stores these as ZADD values directly.
+
+User: Add a unit test for the delay schedule.
+
+Agent: [Write: tests/unit/delivery/retry-queue.test.ts]
+
+Test covers: first enqueue sets correct score within 100ms tolerance, fifth attempt goes to dead letter, pollDueRetries returns jobs whose score is in the past, removeFromQueue removes only the target job.
+
+User: Run it.
+
+Agent: [Bash: npm test -- tests/unit/delivery/retry-queue.test.ts]
+
+```
+PASS tests/unit/delivery/retry-queue.test.ts
+  enqueueRetry
+    ✓ schedules retry with exponential delay (12ms)
+    ✓ moves to dead letter after MAX_ATTEMPTS (8ms)
+  pollDueRetries
+    ✓ returns only due jobs (5ms)
+  removeFromQueue
+    ✓ removes target job only (4ms)
+```
+
+All passing.