@aitne/daemon 0.1.7 → 0.1.9

package/dist/api/routes/observations.js

@@ -1,9 +1,10 @@
 import { Hono } from "hono";
-import { INTEGRATION_KEYS, isIntegrationKey, } from "@aitne/shared";
 import { consumeObservations, getNoveltyDistribution, getObservationStats, getPendingObservations, getSummaryStatusCounts, recordObservation, } from "../../db/observations.js";
 import { readIntegrationFlipLock } from "../../core/integration-lifecycle.js";
 import { createLogger } from "../../logging.js";
 import { DEFAULT_JSON_BODY_MAX_BYTES, readJsonBody } from "../json-body.js";
+import { composeIssue, respondWithAgentError } from "../helpers/agent-errors.js";
+import { BATCH_EXAMPLE, BATCH_EXPECTED_SHAPE, BATCH_MAX_OBSERVATIONS, inferIntegrationKeyFromSource as inferIntegrationKeyFromSourceShared, normalizeMailObservationPayload as normalizeMailObservationPayloadShared, processObservationsBatch, } from "../../services/observations-batch.js";
 const logger = createLogger("observations-api");
 function parseBoolean(value, defaultValue) {
     if (value === undefined)
@@ -25,7 +26,7 @@ function parsePayload(payload) {
     }
 }
 /**
- * ROUTINE_DATA_ACQUISITION_DESIGN.md §6.7 — turn the comma-separated
+ * docs/design/appendices/routine-data-acquisition.md §6.7 — turn the comma-separated
  * `?source_prefix=gmail:,outlook_mail:` query string into a list the
  * db helper can OR via per-prefix `LIKE` predicates. Returns
  * `undefined` (not `[]`) when the param is absent so the db helper can
@@ -46,7 +47,7 @@ function parseSourcePrefixes(raw) {
     return parts.length > 0 ? parts : undefined;
 }
 /**
- * ROUTINE_DATA_ACQUISITION_DESIGN.md CR2 — resolve `since=` /
+ * docs/design/appendices/routine-data-acquisition.md CR2 — resolve `since=` /
  * `observed_at_after=` from the request, validate the format, and
  * coerce empty strings to `undefined` so the db filter is skipped
  * rather than emitted as `datetime(observed_at) >= datetime("")`
@@ -89,15 +90,14 @@ function resolveSinceParam(rawSince, rawAlias) {
  * `"gmail:account-1"`) so per-account / per-database source values still
  * resolve. Returns null for anything else.
  */
-function inferIntegrationKeyFromSource(source) {
-    if (isIntegrationKey(source))
-        return source;
-    for (const key of INTEGRATION_KEYS) {
-        if (source.startsWith(`${key}:`))
-            return key;
-    }
-    return null;
-}
+// inferIntegrationKeyFromSource + normalizeMailObservationPayload moved to
+// `services/observations-batch.ts` so the SDK MCP tool
+// `mcp__aitne-observations__submit_observations` can reuse them. The route
+// re-exports the imports under the same local names so the rest of this
+// file (POST /observations, the legacy /observations/batch envelope code,
+// metric helpers below) reads identically.
+const inferIntegrationKeyFromSource = inferIntegrationKeyFromSourceShared;
+const normalizeMailObservationPayload = normalizeMailObservationPayloadShared;
 /**
  * cost-reduction-structural §A "Failure modes" — the summary may be
  * outdated when the worker lags far behind the observation moment (e.g.
@@ -141,7 +141,7 @@ export function createObservationRoutes(deps) {
         const source = c.req.query("source");
         const sourcePrefixRaw = c.req.query("source_prefix");
         const actor = c.req.query("actor");
-        // ROUTINE_DATA_ACQUISITION_DESIGN.md §6.7 / CR2 — `since=` is the
+        // docs/design/appendices/routine-data-acquisition.md §6.7 / CR2 — `since=` is the
         // canonical name; `observed_at_after=` is the routine-side alias
         // (`since` wins when both are supplied). Empty / whitespace-only
         // values are coerced to "no filter" rather than `datetime("")
@@ -158,7 +158,12 @@ export function createObservationRoutes(deps) {
             }, 400);
         }
         if (actor && !["user", "agent", "system", "unknown"].includes(actor)) {
-            return c.json({ error: "invalid_actor" }, 400);
+            return respondWithAgentError(c, 400, [
+                composeIssue("observations.invalid_actor", {
+                    field: "actor",
+                    received: actor,
+                }),
+            ]);
         }
         const sourceFilterPrefixes = parseSourcePrefixes(sourcePrefixRaw);
         const observations = getPendingObservations(db, {
@@ -379,12 +384,13 @@ export function createObservationRoutes(deps) {
                 }, 409);
             }
         }
+        const normalizedPayload = normalizeMailObservationPayload(source, body.payload);
         const result = recordObservation(db, {
             source,
             ref,
             changeType: changeType,
             actor: actor,
-            payload: body.payload,
+            payload: normalizedPayload,
         });
         logger.info({
             source,
@@ -393,7 +399,7 @@ export function createObservationRoutes(deps) {
             contentHash: result.contentHash,
             action: result.action,
         }, "Observation recorded via API");
-        // ROUTINE_DATA_ACQUISITION_DESIGN.md CR1 — surface payload-identical
+        // docs/design/appendices/routine-data-acquisition.md CR1 — surface payload-identical
         // re-posts as 409 so the routine pre-pass fetcher's JSON return
         // shape (`{"fetched":N,"posted":M,"duplicates":K,"errors":[…]}`)
         // can count them. The 409 body's `error` field distinguishes this
@@ -431,11 +437,11 @@ export function createObservationRoutes(deps) {
      *     batching shape blocks with "curl command must contain an explicit
      *     localhost URL" — the URL lives in the stdin payload, not argv.
      *
-     * Production telemetry on 2026-05-13 morning routine showed Haiku
-     * burning four budget cycles and posting zero observations because every
-     * batching shape it tried was blocked, leaving `today.md` empty. The
-     * single-curl-with-array endpoint resolves the cardinality mismatch
-     * without weakening either hook.
+     * Without this single-curl-with-array endpoint Haiku burns budget
+     * cycles trying every batching shape (each blocked by one of the hooks)
+     * and posts zero observations, leaving `today.md` empty. The bulk
+     * endpoint resolves the cardinality mismatch without weakening either
+     * hook.
      *
      * Body: `{ "observations": [...] }` with up to 200 entries per call.
      * Per-item validation mirrors POST /observations exactly. The whole
@@ -446,80 +452,13 @@ export function createObservationRoutes(deps) {
      * outcomes live in `results[*].status` so the agent doesn't retry the
      * whole batch on a partial failure.
      */
-    const BATCH_MAX_OBSERVATIONS = 200;
-    const BATCH_EXPECTED_SHAPE = '{"observations": [{"source": string, "ref": string, "changeType"?: "created"|"modified"|"deleted", "actor"?: "agent"|"system", "payload"?: unknown}, ...]}';
-    const BATCH_EXAMPLE = '{"observations":[{"source":"google_calendar:primary","ref":"evt-1","payload":{"kind":"calendar","providerId":"primary","raw":{"title":"…"}}},{"source":"google_calendar:primary","ref":"evt-2","payload":{"kind":"calendar","providerId":"primary","raw":{"title":"…"}}}]}';
-    function validateBatchItem(item, index) {
-        if (!item || typeof item !== "object" || Array.isArray(item)) {
-            return {
-                ok: false,
-                result: {
-                    index,
-                    status: "validation_error",
-                    error: "item must be a JSON object",
-                    hint: BATCH_EXPECTED_SHAPE,
-                },
-            };
-        }
-        const obj = item;
-        if (typeof obj.source !== "string" || obj.source.length === 0) {
-            return {
-                ok: false,
-                result: {
-                    index,
-                    status: "validation_error",
-                    error: "'source' must be a non-empty string",
-                    hint: "Use 'gmail:<account>', 'google_calendar:<calendarId>', 'notion:<dbId>', etc.",
-                },
-            };
-        }
-        if (typeof obj.ref !== "string" || obj.ref.length === 0) {
-            return {
-                ok: false,
-                result: {
-                    index,
-                    status: "validation_error",
-                    source: obj.source,
-                    error: "'ref' must be a non-empty string",
-                    hint: "Stable id within the source — e.g. message id, event id",
-                },
-            };
-        }
-        const changeType = typeof obj.changeType === "string" ? obj.changeType : "created";
-        if (!["created", "modified", "deleted"].includes(changeType)) {
-            return {
-                ok: false,
-                result: {
-                    index,
-                    status: "validation_error",
-                    source: obj.source,
-                    ref: obj.ref,
-                    error: `'changeType' must be 'created'|'modified'|'deleted' — received '${changeType}'`,
-                },
-            };
-        }
-        const actor = typeof obj.actor === "string" ? obj.actor : "agent";
-        if (!["agent", "system"].includes(actor)) {
-            return {
-                ok: false,
-                result: {
-                    index,
-                    status: "validation_error",
-                    source: obj.source,
-                    ref: obj.ref,
-                    error: `'actor' must be 'agent' or 'system' — received '${actor}'`,
-                },
-            };
-        }
-        return {
-            ok: true,
-            source: obj.source,
-            ref: obj.ref,
-            changeType: changeType,
-            actor: actor,
-            payload: obj.payload,
-        };
-    }
+    // BATCH_MAX_OBSERVATIONS / BATCH_EXPECTED_SHAPE / BATCH_EXAMPLE /
+    // BatchItemResult / validateBatchItem moved to
+    // `services/observations-batch.ts` so the SDK MCP tool
+    // `mcp__aitne-observations__submit_observations` and this HTTP route
+    // share the same per-item validation + recordObservation loop. The
+    // route keeps only the HTTP-envelope concerns (content-length cap,
+    // JSON parse, batch-too-large 400) below.
     app.post("/observations/batch", async (c) => {
         // Size-cap defense mirrors POST /observations.
         const declared = c.req.header("content-length");
@@ -539,7 +478,12 @@ export function createObservationRoutes(deps) {
         }
         catch (err) {
             const detail = err instanceof Error ? err.message : String(err);
-            return c.json({ error: "invalid_json_body", message: detail }, 400);
+            return respondWithAgentError(c, 400, [
+                composeIssue("observations.invalid_json_body", {
+                    field: "body",
+                    received: detail,
+                }),
+            ], { legacyFields: { message: detail } });
         }
         const actualBytes = Buffer.byteLength(raw, "utf-8");
         if (actualBytes > DEFAULT_JSON_BODY_MAX_BYTES) {
@@ -598,100 +542,26 @@ export function createObservationRoutes(deps) {
                 hint: "Split the batch into chunks of at most 200 items.",
             }, 400);
         }
-        const results = [];
-        let posted = 0;
-        let duplicates = 0;
-        let errorCount = 0;
-        // Single explicit transaction wraps the whole batch — better-sqlite3
-        // transactions amortise the per-statement overhead to ~30-100x for
-        // bulk inserts, which is the primary perf win this endpoint offers
-        // beyond bypassing the bashCurlHook one-curl cap. The flip-lock check
-        // stays per-item so a mixed-integration batch (defensive — the
-        // pre-pass scopes each sub-session to one integration, but the
-        // endpoint must hold under any caller) does not block on the first
-        // unrelated lock.
-        const writeBatch = db.transaction((items) => {
-            for (let i = 0; i < items.length; i++) {
-                const validated = validateBatchItem(items[i], i);
-                if (!validated.ok) {
-                    results.push(validated.result);
-                    errorCount += 1;
-                    continue;
-                }
-                const lockedKey = inferIntegrationKeyFromSource(validated.source);
-                if (lockedKey) {
-                    const lock = readIntegrationFlipLock(db, lockedKey);
-                    if (lock) {
-                        logger.warn({ source: validated.source, ref: validated.ref, lockedKey, lock }, "Batch observation write rejected — integration flip lock held");
-                        results.push({
-                            index: i,
-                            status: "flip_locked",
-                            source: validated.source,
-                            ref: validated.ref,
-                            error: `Integration '${lockedKey}' flip in progress`,
-                        });
-                        errorCount += 1;
-                        continue;
-                    }
-                }
-                const result = recordObservation(db, {
-                    source: validated.source,
-                    ref: validated.ref,
-                    changeType: validated.changeType,
-                    actor: validated.actor,
-                    payload: validated.payload,
-                });
-                if (result.action === "duplicate") {
-                    duplicates += 1;
-                    results.push({
-                        index: i,
-                        status: "duplicate",
-                        source: validated.source,
-                        ref: validated.ref,
-                        contentHash: result.contentHash,
-                        id: result.id,
-                    });
-                }
-                else {
-                    posted += 1;
-                    results.push({
-                        index: i,
-                        status: result.action,
-                        source: validated.source,
-                        ref: validated.ref,
-                        contentHash: result.contentHash,
-                        id: result.id,
-                    });
-                }
-            }
-        });
-        writeBatch(envelope.observations);
+        const batchResult = processObservationsBatch(db, envelope.observations);
         logger.info({
             count: envelope.observations.length,
-            posted,
-            duplicates,
-            errors: errorCount,
+            posted: batchResult.posted,
+            duplicates: batchResult.duplicates,
+            errors: batchResult.errors,
         }, "Observations batch recorded via API");
-        return c.json({
-            results,
-            fetched: envelope.observations.length,
-            posted,
-            duplicates,
-            errors: errorCount,
-        });
+        return c.json(batchResult);
     });
     /**
      * Field-level validation contract for `POST /observations/consume`.
      *
-     * Production telemetry (2026-05) showed a single Stage-3 hourly_check
-     * burning $0.58 / 25 turns retrying this endpoint with shape variants
+     * Without per-field error envelopes, a single Stage-3 hourly_check can
+     * burn turns retrying this endpoint with shape variants
      * (`correlation_id` snake_case, stringified ids, the angle-bracket
      * placeholder copied verbatim, per-id paths, etc.). The legacy
-     * `{ error: "validation_error" }` response gave the agent zero signal
-     * about which field was wrong, so it would mutate a random field and
-     * retry. Returning the full schema + the specific issue + a one-line
-     * hint lets the agent self-correct on the next turn instead of the
-     * eighth.
+     * `{ error: "validation_error" }` response gives the agent zero signal
+     * about which field was wrong, so it mutates a random field and retries.
+     * Returning the full schema + the specific issue + a one-line hint lets
+     * the agent self-correct on the next turn instead of the eighth.
      */
     const CONSUME_EXPECTED_SHAPE = '{"ids": number[], "correlationId": string}';
     const CONSUME_EXAMPLE = '{"ids":[14,17],"correlationId":"hourly-2026-04-23T15:00:00Z-7af3"}';
@@ -795,8 +665,8 @@ export function createObservationRoutes(deps) {
         return c.json(result);
     });
     /**
-     * Helpful 405 for the per-id consume shape the agent has reached for in
-     * production (`POST /api/observations/:id/consume`). Without an explicit
+     * Helpful 405 for the per-id consume shape the agent commonly reaches
+     * for (`POST /api/observations/:id/consume`). Without an explicit
      * handler this path falls through to Hono's 404 with body
      * `"404 Not Found"`, which gives the agent nothing to act on. Returning
      * a 405 with the canonical bulk-endpoint hint pulls the agent back onto
@@ -840,4 +710,3 @@ export function createObservationRoutes(deps) {
     });
     return app;
 }
-//# sourceMappingURL=observations.js.map

package/dist/api/routes/obsidian.d.ts

@@ -13,4 +13,3 @@ export interface ObsidianRouteDependencies {
     writeTracker?: AgentWriteTracker;
 }
 export declare function createObsidianRoutes(deps: ObsidianRouteDependencies): Hono;
-//# sourceMappingURL=obsidian.d.ts.map