memhtml 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +114 -245
- package/agent/instructions.md +74 -84
- package/dist/{dist-CSo_XRfz.mjs → dist-BCsav-EP.mjs} +341 -6
- package/dist/dist-BCsav-EP.mjs.map +1 -0
- package/dist/{dist-B3yDga97.mjs → dist-D5DlgqH2.mjs} +2245 -104
- package/dist/dist-D5DlgqH2.mjs.map +1 -0
- package/dist/dist-DuzGralO.mjs +3 -0
- package/dist/memhtml-mcp.mjs +43 -5
- package/dist/memhtml-mcp.mjs.map +1 -1
- package/dist/memhtml.mjs +62 -9
- package/dist/memhtml.mjs.map +1 -1
- package/package.json +1 -1
- package/src/client.ts +180 -8
- package/src/contract.ts +232 -5
- package/dist/dist-B3yDga97.mjs.map +0 -1
- package/dist/dist-CSo_XRfz.mjs.map +0 -1
- package/dist/dist-FgNX_rxc.mjs +0 -3
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { At as filenameFor, Ct as PERSON_ENTITY_PREFIX, Dt as isTaskStatus, Et as WRITABLE_MEMORY_TYPES, G as InvalidMemory, J as PathNotFound, K as LlmContractViolation, Mt as withCollisionOrdinal, Ot as parseEntity, St as PARA_BUCKETS, Tt as TaskStatus, W as DirtyTree, X as WriteConflict, Y as StorageFailure, Z as EdgeRel, _t as Confidence, at as ARCS_DIR, bt as MemoryStatus, ct as PEOPLE_DIR, dt as isArchivePath, et as isEdgeRel, ft as isValidMemoryPath, gt as placementFor, ht as paraBucketOf, it as ARCHIVE_BUCKET, jt as slugify, kt as SLUG_FALLBACK, mt as normalizePath, nt as relForToken, ot as INBOX_DIR, pt as memoryPathFor, q as ModelUnavailable, rt as relTokenFor, st as MEMORY_EXTENSION, tt as relClassFor, ut as archivePathFor, vt as Importance, wt as TASK_STATUSES, xt as MemoryType, yt as MEMORY_TYPES } from "./dist-BCsav-EP.mjs";
|
|
2
2
|
import { createRequire } from "node:module";
|
|
3
3
|
import { Config, Context, Effect, Layer, Result, Schedule, Schema } from "effect";
|
|
4
4
|
import { mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
|
|
@@ -1673,7 +1673,7 @@ const articleFor = (spec) => {
|
|
|
1673
1673
|
* ordinals a multiple of 180 apart state facts about different environments. It carries no digit, so
|
|
1674
1674
|
* the numeric divergence family still compares exactly the quantity the claim asserts.
|
|
1675
1675
|
*/
|
|
1676
|
-
const claimFor = (topic, type, ordinal) => {
|
|
1676
|
+
const claimFor$1 = (topic, type, ordinal) => {
|
|
1677
1677
|
const noun = topic.nouns[nounIndexOf(ordinal, topic.nouns.length)] ?? "surface";
|
|
1678
1678
|
const verb = topic.verbs[ordinal % topic.verbs.length] ?? "settles";
|
|
1679
1679
|
const scope = scopeFor(ordinal);
|
|
@@ -1859,8 +1859,8 @@ const buildBase = (count, seed) => {
|
|
|
1859
1859
|
const topic = TOPICS[ordinal % TOPICS.length];
|
|
1860
1860
|
const type = TYPES[ordinal % TYPES.length];
|
|
1861
1861
|
const day = 30 + ordinal % 240;
|
|
1862
|
-
const claim = claimFor(topic, type, ordinal);
|
|
1863
|
-
const title = titleFor$
|
|
1862
|
+
const claim = claimFor$1(topic, type, ordinal);
|
|
1863
|
+
const title = titleFor$2(topic, type, ordinal);
|
|
1864
1864
|
const kit = KITS[ordinal % KITS.length];
|
|
1865
1865
|
const jitter = next();
|
|
1866
1866
|
specs.push({
|
|
@@ -1890,7 +1890,7 @@ const buildBase = (count, seed) => {
|
|
|
1890
1890
|
* one subject. `title` is the first field of `fts_text`, and a title naming a different noun than the
|
|
1891
1891
|
* claim would put a term into the lexical arm that the memory does not assert.
|
|
1892
1892
|
*/
|
|
1893
|
-
const titleFor$
|
|
1893
|
+
const titleFor$2 = (topic, type, ordinal) => {
|
|
1894
1894
|
const noun = topic.nouns[nounIndexOf(ordinal, topic.nouns.length)] ?? "surface";
|
|
1895
1895
|
return `${topic.service} ${noun} ${scopeFor(ordinal)} ${type.replace("_", " ")} ${ordinal}`;
|
|
1896
1896
|
};
|
|
@@ -3624,6 +3624,26 @@ const setMeta = (html, name, value) => {
|
|
|
3624
3624
|
return offset === void 0 ? html : insertAt(html, offset, `${metaLine$1(name, value)}\n`);
|
|
3625
3625
|
};
|
|
3626
3626
|
/**
|
|
3627
|
+
* Append another `<meta>` of a repeatable name, a new `memhtml-entity` or `memhtml-tag`, after the
|
|
3628
|
+
* last one already present. Adding a value that is already there is a no-op, so the operation
|
|
3629
|
+
* is idempotent and a re-run of a sleep phase cannot grow the head.
|
|
3630
|
+
*/
|
|
3631
|
+
const addMeta = (html, name, value) => {
|
|
3632
|
+
if (!isMemoryMetaName(name)) return html;
|
|
3633
|
+
const document = parseDocument(html);
|
|
3634
|
+
const head = headOf$1(document);
|
|
3635
|
+
if (head === void 0) return html;
|
|
3636
|
+
const present = memhtmlMetas(head).filter((meta) => meta.name === name);
|
|
3637
|
+
if (present.some((meta) => attr(meta.element, "content") === value)) return html;
|
|
3638
|
+
const last = present.at(-1);
|
|
3639
|
+
if (last !== void 0) {
|
|
3640
|
+
const span = spanOf(last.element);
|
|
3641
|
+
if (span !== void 0) return insertAt(html, lineEndAt(html, span.end), `${metaLine$1(name, value)}\n`);
|
|
3642
|
+
}
|
|
3643
|
+
const offset = insertionOffsetForMeta(html, head, name);
|
|
3644
|
+
return offset === void 0 ? html : insertAt(html, offset, `${metaLine$1(name, value)}\n`);
|
|
3645
|
+
};
|
|
3646
|
+
/**
|
|
3627
3647
|
* Append a `<link rel="memhtml-…">` edge. Idempotent on the `(rel, href)` pair, because the sleep
|
|
3628
3648
|
* conflict phase promotes the same corroborated edge on every run and a duplicated `<link>`
|
|
3629
3649
|
* would become a duplicated `edges` row.
|
|
@@ -5998,7 +6018,7 @@ const MEMORY_BODY_BUDGET = 16e3;
|
|
|
5998
6018
|
*/
|
|
5999
6019
|
const MAX_PER_ENTITY = 2;
|
|
6000
6020
|
/** `arc` memories take {@link ARC_BODY_BUDGET}; everything else takes {@link MEMORY_BODY_BUDGET}. */
|
|
6001
|
-
const budgetFor = (memoryType) => memoryType === "arc" ? ARC_BODY_BUDGET : MEMORY_BODY_BUDGET;
|
|
6021
|
+
const budgetFor$1 = (memoryType) => memoryType === "arc" ? ARC_BODY_BUDGET : MEMORY_BODY_BUDGET;
|
|
6002
6022
|
/**
|
|
6003
6023
|
* Fold ranked candidates into quotes and index lines under one character budget.
|
|
6004
6024
|
*
|
|
@@ -7746,7 +7766,7 @@ const makeRetrieval = (deps) => {
|
|
|
7746
7766
|
* every concrete memory behind it. The pack would then explain the pattern and cite none of the
|
|
7747
7767
|
* evidence.
|
|
7748
7768
|
*/
|
|
7749
|
-
const arcs = foldDisclosure(candidates.filter((candidate) => candidate.memoryType === "arc"), budgetFor("arc"));
|
|
7769
|
+
const arcs = foldDisclosure(candidates.filter((candidate) => candidate.memoryType === "arc"), budgetFor$1("arc"));
|
|
7750
7770
|
const memories = foldDisclosure(candidates.filter((candidate) => candidate.memoryType !== "arc"), budget);
|
|
7751
7771
|
return {
|
|
7752
7772
|
arcs,
|
|
@@ -8247,12 +8267,23 @@ const EmbeddingsLive = Layer.effect(Embeddings, Effect.gen(function* () {
|
|
|
8247
8267
|
//#endregion
|
|
8248
8268
|
//#region packages/llm/dist/models.js
|
|
8249
8269
|
/**
|
|
8250
|
-
* The
|
|
8251
|
-
*
|
|
8252
|
-
*
|
|
8253
|
-
*
|
|
8270
|
+
* The models the sleep phases run on, and the wire rules that differ between them.
|
|
8271
|
+
*
|
|
8272
|
+
* Two providers, two call shapes, one decode. The Anthropic lane speaks the native
|
|
8273
|
+
* Messages body with a forced `emit` tool; the OpenAI lane speaks the chat-completions
|
|
8274
|
+
* body with `response_format: {type: "json_schema", strict: true}`. The OpenAI lane
|
|
8275
|
+
* exists for exactly one property the Anthropic lane cannot offer on Bedrock today:
|
|
8276
|
+
* constrained decoding, which makes an off-schema structured answer impossible at
|
|
8277
|
+
* generation time instead of repaired after (probed live 2026-08-22 — the Claude 5
|
|
8278
|
+
* models reject `strict` and `output_config.format` on every Bedrock surface, while
|
|
8279
|
+
* `global.openai.gpt-5.6-sol` honors strict JSON schema on InvokeModel). Both lanes
|
|
8280
|
+
* converge on the same response shape before `decodeToolInput`, so every phase and
|
|
8281
|
+
* every test sees one contract.
|
|
8282
|
+
*/
|
|
8283
|
+
/**
|
|
8284
|
+
* Reasoning effort. The Anthropic lane passes it as `output_config.effort`, the OpenAI
|
|
8285
|
+
* lane as `reasoning_effort`; both accept all four values (sol probed live 2026-08-22).
|
|
8254
8286
|
*/
|
|
8255
|
-
/** Reasoning effort, passed as `output_config.effort`. Accepted by all three models. */
|
|
8256
8287
|
const Effort = Schema.Literals([
|
|
8257
8288
|
"low",
|
|
8258
8289
|
"medium",
|
|
@@ -8262,27 +8293,39 @@ const Effort = Schema.Literals([
|
|
|
8262
8293
|
const ModelKey = Schema.Literals([
|
|
8263
8294
|
"sonnet-5",
|
|
8264
8295
|
"opus-5",
|
|
8265
|
-
"fable-5"
|
|
8296
|
+
"fable-5",
|
|
8297
|
+
"gpt-5.6-sol"
|
|
8266
8298
|
]);
|
|
8267
8299
|
/**
|
|
8268
8300
|
* Bedrock ids use the `global.` inference profiles, which makes them reachable from a
|
|
8269
|
-
* single region without provisioning per-region throughput.
|
|
8301
|
+
* single region without provisioning per-region throughput. The OpenAI models REQUIRE
|
|
8302
|
+
* the profile: the bare `openai.gpt-5.6-*` ids reject on-demand invocation outright
|
|
8303
|
+
* (probed live 2026-08-22).
|
|
8270
8304
|
*/
|
|
8271
8305
|
const MODELS = [
|
|
8272
8306
|
{
|
|
8273
8307
|
key: "sonnet-5",
|
|
8274
8308
|
label: "Claude Sonnet 5",
|
|
8275
|
-
modelId: "global.anthropic.claude-sonnet-5"
|
|
8309
|
+
modelId: "global.anthropic.claude-sonnet-5",
|
|
8310
|
+
provider: "anthropic"
|
|
8276
8311
|
},
|
|
8277
8312
|
{
|
|
8278
8313
|
key: "opus-5",
|
|
8279
8314
|
label: "Claude Opus 5",
|
|
8280
|
-
modelId: "global.anthropic.claude-opus-5"
|
|
8315
|
+
modelId: "global.anthropic.claude-opus-5",
|
|
8316
|
+
provider: "anthropic"
|
|
8281
8317
|
},
|
|
8282
8318
|
{
|
|
8283
8319
|
key: "fable-5",
|
|
8284
8320
|
label: "Claude Fable 5",
|
|
8285
|
-
modelId: "global.anthropic.claude-fable-5"
|
|
8321
|
+
modelId: "global.anthropic.claude-fable-5",
|
|
8322
|
+
provider: "anthropic"
|
|
8323
|
+
},
|
|
8324
|
+
{
|
|
8325
|
+
key: "gpt-5.6-sol",
|
|
8326
|
+
label: "GPT-5.6 Sol",
|
|
8327
|
+
modelId: "global.openai.gpt-5.6-sol",
|
|
8328
|
+
provider: "openai"
|
|
8286
8329
|
}
|
|
8287
8330
|
];
|
|
8288
8331
|
const BY_KEY = new Map(MODELS.map((model) => [model.key, model]));
|
|
@@ -8297,12 +8340,13 @@ const modelByKey = (key) => {
|
|
|
8297
8340
|
return found;
|
|
8298
8341
|
};
|
|
8299
8342
|
/**
|
|
8300
|
-
* The `thinking` object per model. Opus 5 and Fable 5 take `{type: "adaptive"}`
|
|
8301
|
-
* adaptive-only). Sonnet 5 reasons unconditionally and takes NO thinking key.
|
|
8302
|
-
* to Sonnet 5 raises a validation error instead of being ignored.
|
|
8343
|
+
* The `thinking` object per Anthropic model. Opus 5 and Fable 5 take `{type: "adaptive"}`
|
|
8344
|
+
* (Fable is adaptive-only). Sonnet 5 reasons unconditionally and takes NO thinking key.
|
|
8345
|
+
* Sending one to Sonnet 5 raises a validation error instead of being ignored. The OpenAI
|
|
8346
|
+
* lane never consults this: its reasoning dial is `reasoning_effort` alone.
|
|
8303
8347
|
*
|
|
8304
|
-
* Verified live 2026-08-02: all three accept this shape alongside a forced
|
|
8305
|
-
* so structured output and adaptive thinking compose.
|
|
8348
|
+
* Verified live 2026-08-02: all three Claude models accept this shape alongside a forced
|
|
8349
|
+
* `tool_choice`, so structured output and adaptive thinking compose.
|
|
8306
8350
|
*/
|
|
8307
8351
|
const thinkingFor = (key) => key === "opus-5" || key === "fable-5" ? { type: "adaptive" } : null;
|
|
8308
8352
|
|
|
@@ -8317,6 +8361,13 @@ const thinkingFor = (key) => key === "opus-5" || key === "fable-5" ? { type: "ad
|
|
|
8317
8361
|
* default for an omitted field, and no accepted extra key. Downstream code cannot tell a
|
|
8318
8362
|
* coerced object from a real one, and the phases that consume these objects archive and
|
|
8319
8363
|
* rewrite files.
|
|
8364
|
+
*
|
|
8365
|
+
* ONE repair is the exception, because it recovers the payload the model meant rather than
|
|
8366
|
+
* inventing one: a top-level field the schema declares as an array or object sometimes
|
|
8367
|
+
* arrives double-encoded as a JSON STRING. That string is parsed once and the SAME strict
|
|
8368
|
+
* decode re-runs on the result, so nothing an off-schema answer carries can slip through —
|
|
8369
|
+
* a payload the repair cannot make satisfy the schema still fails with the original
|
|
8370
|
+
* violation. See {@link decodeToolInput}.
|
|
8320
8371
|
*/
|
|
8321
8372
|
/** Cap on the raw payload carried on a violation, so a runaway response cannot bloat it. */
|
|
8322
8373
|
const MAX_RAW = 800;
|
|
@@ -8351,6 +8402,50 @@ const preview = (payload) => {
|
|
|
8351
8402
|
})();
|
|
8352
8403
|
return rendered.length <= 800 ? rendered : `${rendered.slice(0, 800)}…`;
|
|
8353
8404
|
};
|
|
8405
|
+
/** True when a property's derived JSON schema declares a container: an array, an object, or a
|
|
8406
|
+
* `$ref` (every hoisted definition is a struct). A string-typed property is NOT a container,
|
|
8407
|
+
* which is what keeps a field that legitimately holds JSON-looking text out of the repair. */
|
|
8408
|
+
const expectsContainer = (property) => {
|
|
8409
|
+
if (typeof property !== "object" || property === null) return false;
|
|
8410
|
+
const record = property;
|
|
8411
|
+
return typeof record.$ref === "string" || record.type === "array" || record.type === "object";
|
|
8412
|
+
};
|
|
8413
|
+
/**
|
|
8414
|
+
* Undo ONE level of JSON-string double-encoding on the top-level fields of a tool payload.
|
|
8415
|
+
*
|
|
8416
|
+
* The shape this repairs was observed on the wire: a field the schema declares as an array
|
|
8417
|
+
* arrives as `"{\"groups\":[…]}"` — the whole answer serialized as a string under its own
|
|
8418
|
+
* key — or as the array itself serialized. So a string sitting where the derived
|
|
8419
|
+
* `input_schema` declares a container is parsed once; when the parsed value is an object
|
|
8420
|
+
* carrying the SAME key, the value at that key is taken, otherwise the parsed value stands.
|
|
8421
|
+
*
|
|
8422
|
+
* Returns `undefined` when there is nothing to repair: no field qualified, or no parse
|
|
8423
|
+
* succeeded. The caller then reports the ORIGINAL violation, and a repaired payload still
|
|
8424
|
+
* re-runs the same strict decode, so this never widens what the schema accepts.
|
|
8425
|
+
*/
|
|
8426
|
+
const unwrapDoubleEncoded = (schema, input) => {
|
|
8427
|
+
if (typeof input !== "object" || input === null || Array.isArray(input)) return void 0;
|
|
8428
|
+
const properties = toInputSchema(schema).properties;
|
|
8429
|
+
if (properties === void 0) return void 0;
|
|
8430
|
+
let repairedAField = false;
|
|
8431
|
+
const repaired = {};
|
|
8432
|
+
for (const [key, received] of Object.entries(input)) {
|
|
8433
|
+
repaired[key] = received;
|
|
8434
|
+
if (typeof received !== "string" || !expectsContainer(properties[key])) continue;
|
|
8435
|
+
const parsed = (() => {
|
|
8436
|
+
try {
|
|
8437
|
+
return { value: JSON.parse(received) };
|
|
8438
|
+
} catch {
|
|
8439
|
+
return;
|
|
8440
|
+
}
|
|
8441
|
+
})();
|
|
8442
|
+
if (parsed === void 0) continue;
|
|
8443
|
+
const wrapper = typeof parsed.value === "object" && parsed.value !== null && !Array.isArray(parsed.value) ? parsed.value : void 0;
|
|
8444
|
+
repaired[key] = wrapper !== void 0 && key in wrapper ? wrapper[key] : parsed.value;
|
|
8445
|
+
repairedAField = true;
|
|
8446
|
+
}
|
|
8447
|
+
return repairedAField ? repaired : void 0;
|
|
8448
|
+
};
|
|
8354
8449
|
/**
|
|
8355
8450
|
* Decode a forced-tool payload against its schema.
|
|
8356
8451
|
*
|
|
@@ -8359,13 +8454,29 @@ const preview = (payload) => {
|
|
|
8359
8454
|
* would let a model answer a schema next to the one it was given and have the extra field
|
|
8360
8455
|
* vanish. croq's judge rules out the same drift by enumerating its allowed keys.
|
|
8361
8456
|
*
|
|
8457
|
+
* One failure shape is repaired before the violation is constructed: a top-level container
|
|
8458
|
+
* field double-encoded as a JSON string ({@link unwrapDoubleEncoded}). The repaired payload
|
|
8459
|
+
* goes through the SAME strict decode, and a repair that still does not satisfy the schema
|
|
8460
|
+
* reports the original payload's violation, so the repair cannot mask a genuinely off-schema
|
|
8461
|
+
* answer.
|
|
8462
|
+
*
|
|
8362
8463
|
* `undefined` input means the model produced no `emit` call at all. That is the same class
|
|
8363
8464
|
* of failure as a malformed one, and the reason text names it so a caller can tell the two
|
|
8364
8465
|
* apart in a log without a second error type.
|
|
8365
8466
|
*/
|
|
8366
8467
|
const decodeToolInput = (schema, input) => input === void 0 ? Effect.fail(LlmContractViolation.make({ reason: "model returned no tool_use block for the forced tool" })) : Effect.gen(function* () {
|
|
8367
|
-
const
|
|
8368
|
-
|
|
8468
|
+
const strictDecode = Schema.decodeUnknownEffect(schema, { onExcessProperty: "error" });
|
|
8469
|
+
const decoded = yield* Effect.result(strictDecode(input));
|
|
8470
|
+
if (Result.isSuccess(decoded)) return decoded.success;
|
|
8471
|
+
const repaired = unwrapDoubleEncoded(schema, input);
|
|
8472
|
+
if (repaired !== void 0) {
|
|
8473
|
+
const redecoded = yield* Effect.result(strictDecode(repaired));
|
|
8474
|
+
if (Result.isSuccess(redecoded)) {
|
|
8475
|
+
yield* Effect.logWarning("llm.structured repaired a double-encoded tool field before decoding");
|
|
8476
|
+
return redecoded.success;
|
|
8477
|
+
}
|
|
8478
|
+
}
|
|
8479
|
+
return yield* Effect.fail(LlmContractViolation.make({ reason: `tool payload does not satisfy its schema: ${String(decoded.failure)} (raw: ${preview(input)})` }));
|
|
8369
8480
|
});
|
|
8370
8481
|
|
|
8371
8482
|
//#endregion
|
|
@@ -8377,16 +8488,18 @@ const decodeToolInput = (schema, input) => input === void 0 ? Effect.fail(LlmCon
|
|
|
8377
8488
|
*/
|
|
8378
8489
|
const clampTokens = (requested) => Math.min(requested ?? 16384, MAX_TOKENS_CEILING);
|
|
8379
8490
|
/**
|
|
8380
|
-
* Build the request body. The `tool` argument selects the lane.
|
|
8381
|
-
* answers in prose. When it is present
|
|
8382
|
-
* `emit` call
|
|
8491
|
+
* Build the request body in the model's own dialect. The `tool` argument selects the lane.
|
|
8492
|
+
* When it is absent the model answers in prose. When it is present the request constrains
|
|
8493
|
+
* the model to exactly one schema-shaped answer: a forced `emit` tool call on the
|
|
8494
|
+
* Anthropic dialect, a strict `json_schema` response format on the OpenAI one.
|
|
8383
8495
|
*
|
|
8384
8496
|
* `system` is omitted rather than sent empty, because an empty system block is a distinct
|
|
8385
8497
|
* (and rejected) input from no system block at all. An omitted system also has nothing to cache, so
|
|
8386
8498
|
* `cacheSystem` over an absent or empty system emits no `system` key at all instead of an empty
|
|
8387
8499
|
* cached block.
|
|
8388
8500
|
*/
|
|
8389
|
-
const buildInvokeBody = (key, prompt, options, tool) =>
|
|
8501
|
+
const buildInvokeBody = (key, prompt, options, tool) => modelByKey(key).provider === "openai" ? buildOpenAiBody(prompt, options, tool) : buildAnthropicBody(key, prompt, options, tool);
|
|
8502
|
+
const buildAnthropicBody = (key, prompt, options, tool) => {
|
|
8390
8503
|
const body = {
|
|
8391
8504
|
anthropic_version: ANTHROPIC_VERSION,
|
|
8392
8505
|
max_tokens: clampTokens(options.maxTokens),
|
|
@@ -8417,6 +8530,51 @@ const buildInvokeBody = (key, prompt, options, tool) => {
|
|
|
8417
8530
|
return JSON.stringify(body);
|
|
8418
8531
|
};
|
|
8419
8532
|
/**
|
|
8533
|
+
* The OpenAI chat-completions body (probed live 2026-08-22 against
|
|
8534
|
+
* `global.openai.gpt-5.6-sol` — every field here is one the probe exercised).
|
|
8535
|
+
*
|
|
8536
|
+
* Differences from the Anthropic dialect, each one deliberate:
|
|
8537
|
+
*
|
|
8538
|
+
* - The token budget is `max_completion_tokens` and it bounds reasoning and answer
|
|
8539
|
+
* together, so the same clamp applies. 128k accepted at the ceiling.
|
|
8540
|
+
* - Effort is `reasoning_effort`, taking the same four values.
|
|
8541
|
+
* - `system` rides as a leading `{role: "system"}` message; there is no `system` field.
|
|
8542
|
+
* `cacheSystem` has no OpenAI-side marker — Bedrock reports `cache_write_tokens` in
|
|
8543
|
+
* this dialect's usage without an opt-in field — so the flag is accepted and unused
|
|
8544
|
+
* rather than rejected, keeping the option surface identical across lanes.
|
|
8545
|
+
* - The structured mechanism is `response_format.json_schema` with `strict: true`, named
|
|
8546
|
+
* `emit` so logs read the same across providers. `description` becomes the schema's
|
|
8547
|
+
* own `description`, the closest surface this dialect has to a tool description.
|
|
8548
|
+
*/
|
|
8549
|
+
const buildOpenAiBody = (prompt, options, tool) => {
|
|
8550
|
+
const messages = [];
|
|
8551
|
+
if (options.system !== void 0 && options.system.length > 0) messages.push({
|
|
8552
|
+
role: "system",
|
|
8553
|
+
content: options.system
|
|
8554
|
+
});
|
|
8555
|
+
messages.push({
|
|
8556
|
+
role: "user",
|
|
8557
|
+
content: prompt
|
|
8558
|
+
});
|
|
8559
|
+
const body = {
|
|
8560
|
+
max_completion_tokens: clampTokens(options.maxTokens),
|
|
8561
|
+
messages,
|
|
8562
|
+
reasoning_effort: options.effort
|
|
8563
|
+
};
|
|
8564
|
+
if (tool !== void 0) body.response_format = {
|
|
8565
|
+
type: "json_schema",
|
|
8566
|
+
json_schema: {
|
|
8567
|
+
name: STRUCTURED_TOOL_NAME,
|
|
8568
|
+
strict: true,
|
|
8569
|
+
schema: tool.description === void 0 ? tool.inputSchema : {
|
|
8570
|
+
description: tool.description,
|
|
8571
|
+
...tool.inputSchema
|
|
8572
|
+
}
|
|
8573
|
+
}
|
|
8574
|
+
};
|
|
8575
|
+
return JSON.stringify(body);
|
|
8576
|
+
};
|
|
8577
|
+
/**
|
|
8420
8578
|
* `stop_reason` values that mean the content is not a complete answer. Both become typed
|
|
8421
8579
|
* failures. A response cut off at `max_tokens` may never have reached the point that made
|
|
8422
8580
|
* it a judgment, and a refusal carries no judgment at all. Reading either as a finished
|
|
@@ -8425,6 +8583,55 @@ const buildInvokeBody = (key, prompt, options, tool) => {
|
|
|
8425
8583
|
const INCOMPLETE_STOP_REASONS = /* @__PURE__ */ new Set(["max_tokens", "refusal"]);
|
|
8426
8584
|
/** The parsed payload, read defensively, since every field on the wire is optional. */
|
|
8427
8585
|
const asResponseBody = (payload) => payload ?? {};
|
|
8586
|
+
/**
|
|
8587
|
+
* Fold an OpenAI chat-completions payload into {@link InvokeResponseBody}, so one read
|
|
8588
|
+
* side serves both dialects.
|
|
8589
|
+
*
|
|
8590
|
+
* `finish_reason` maps onto the Anthropic vocabulary this module already gates on:
|
|
8591
|
+
* `length` is `max_tokens` and `content_filter` is `refusal`, both of which
|
|
8592
|
+
* {@link INCOMPLETE_STOP_REASONS} already refuses; everything else passes through as a
|
|
8593
|
+
* complete answer. `structured` says how the caller asked, which decides how the content
|
|
8594
|
+
* is presented: a structured request's content is the schema-constrained JSON, parsed
|
|
8595
|
+
* here and presented as the `emit` tool's input, and a prose request's content is a text
|
|
8596
|
+
* block. Content that fails to parse on the structured path yields NO tool block, which
|
|
8597
|
+
* downstream reports as the existing "no tool_use block" violation — the right class,
|
|
8598
|
+
* since constrained decoding makes that a broken response rather than an off-schema one.
|
|
8599
|
+
*/
|
|
8600
|
+
const normalizeOpenAiResponse = (payload, structured) => {
|
|
8601
|
+
const body = payload ?? {};
|
|
8602
|
+
const choice = body.choices?.[0];
|
|
8603
|
+
const finish = choice?.finish_reason ?? null;
|
|
8604
|
+
const stopReason = finish === "length" ? "max_tokens" : finish === "content_filter" ? "refusal" : finish;
|
|
8605
|
+
const text = choice?.message?.content;
|
|
8606
|
+
const content = [];
|
|
8607
|
+
if (typeof text === "string" && text.length > 0) {
|
|
8608
|
+
if (structured) {
|
|
8609
|
+
const input = (() => {
|
|
8610
|
+
try {
|
|
8611
|
+
return { value: JSON.parse(text) };
|
|
8612
|
+
} catch {
|
|
8613
|
+
return;
|
|
8614
|
+
}
|
|
8615
|
+
})();
|
|
8616
|
+
if (input !== void 0) content.push({
|
|
8617
|
+
type: "tool_use",
|
|
8618
|
+
name: STRUCTURED_TOOL_NAME,
|
|
8619
|
+
input: input.value
|
|
8620
|
+
});
|
|
8621
|
+
} else content.push({
|
|
8622
|
+
type: "text",
|
|
8623
|
+
text
|
|
8624
|
+
});
|
|
8625
|
+
}
|
|
8626
|
+
return {
|
|
8627
|
+
stop_reason: stopReason,
|
|
8628
|
+
content,
|
|
8629
|
+
usage: {
|
|
8630
|
+
...body.usage?.prompt_tokens === void 0 ? {} : { input_tokens: body.usage.prompt_tokens },
|
|
8631
|
+
...body.usage?.completion_tokens === void 0 ? {} : { output_tokens: body.usage.completion_tokens }
|
|
8632
|
+
}
|
|
8633
|
+
};
|
|
8634
|
+
};
|
|
8428
8635
|
/** The incomplete `stop_reason`, or null when the response ran to a natural end. */
|
|
8429
8636
|
const incompleteReason = (parsed) => {
|
|
8430
8637
|
const stop = parsed.stop_reason ?? null;
|
|
@@ -8463,7 +8670,7 @@ const makeModelClient = (client) => {
|
|
|
8463
8670
|
description: tool.description
|
|
8464
8671
|
}));
|
|
8465
8672
|
const finished = yield* Effect.clockWith((clock) => clock.currentTimeMillis);
|
|
8466
|
-
const parsed = asResponseBody(payload);
|
|
8673
|
+
const parsed = model.provider === "openai" ? normalizeOpenAiResponse(payload, tool !== void 0) : asResponseBody(payload);
|
|
8467
8674
|
const incomplete = incompleteReason(parsed);
|
|
8468
8675
|
if (incomplete !== null) return yield* Effect.fail(ModelUnavailable.make({
|
|
8469
8676
|
modelId: model.modelId,
|
|
@@ -8949,12 +9156,21 @@ const batchCall = (model, label, request) => isolate(label, model.generateObject
|
|
|
8949
9156
|
* `sleep_phases` row, and in a `--phases` flag, and three copies of the string would drift.
|
|
8950
9157
|
*/
|
|
8951
9158
|
/**
|
|
8952
|
-
* The
|
|
9159
|
+
* The sixteen phases, in execution order.
|
|
8953
9160
|
*
|
|
8954
9161
|
* The order encodes the predecessor memory system's dependencies (design §6): entity resolution precedes person
|
|
8955
9162
|
* links so aliases have already merged, confidence decay precedes retention triage so triage
|
|
8956
9163
|
* scores the decayed value, and dedup-merge precedes compress and retention because both operate
|
|
8957
9164
|
* on the post-merge set.
|
|
9165
|
+
*
|
|
9166
|
+
* `task-detection` is sixteenth-in-list and thirteenth-in-order, sitting after `trace-consolidation`
|
|
9167
|
+
* and before `integrity`, and both edges are deliberate. It scans the ACTIVE corpus for unresolved
|
|
9168
|
+
* commitments, so it has to run after every phase that changes what is active — after dedup's folds,
|
|
9169
|
+
* after retention's evictions, after compress's canonicals, and after trace consolidation's newly
|
|
9170
|
+
* distilled memories, which are the freshest text of the night and the likeliest to carry one. And it
|
|
9171
|
+
* WRITES files, so it must precede `integrity`, which repairs dangling hrefs and regenerates the
|
|
9172
|
+
* directory artifacts: a task minted afterwards would be absent from its directory's `index.html`
|
|
9173
|
+
* until the next night.
|
|
8958
9174
|
*/
|
|
8959
9175
|
const SLEEP_PHASES = [
|
|
8960
9176
|
"preflight",
|
|
@@ -8969,6 +9185,7 @@ const SLEEP_PHASES = [
|
|
|
8969
9185
|
"compress",
|
|
8970
9186
|
"reprieve",
|
|
8971
9187
|
"trace-consolidation",
|
|
9188
|
+
"task-detection",
|
|
8972
9189
|
"integrity",
|
|
8973
9190
|
"state-export",
|
|
8974
9191
|
"report"
|
|
@@ -9116,6 +9333,18 @@ const meta = (name, value) => ({
|
|
|
9116
9333
|
name,
|
|
9117
9334
|
value
|
|
9118
9335
|
});
|
|
9336
|
+
/**
|
|
9337
|
+
* An APPEND to a repeatable meta, as a value. `setMeta` writes the first value of a name and cannot
|
|
9338
|
+
* express a second, so a phase adding one more `memhtml-tag` needs this constructor.
|
|
9339
|
+
*
|
|
9340
|
+
* Idempotent, because `addMeta` returns the input unchanged when the value is already present — which is
|
|
9341
|
+
* what keeps a re-run of a phase that appends a tag free, the same property `stampFile` reads.
|
|
9342
|
+
*/
|
|
9343
|
+
const addTag = (value) => ({
|
|
9344
|
+
kind: "addMeta",
|
|
9345
|
+
name: "memhtml-tag",
|
|
9346
|
+
value
|
|
9347
|
+
});
|
|
9119
9348
|
/** A link addition, as a value. */
|
|
9120
9349
|
const link = (rel, href) => ({
|
|
9121
9350
|
kind: "addLink",
|
|
@@ -9135,6 +9364,7 @@ const unlink = (rel, href) => href === void 0 ? {
|
|
|
9135
9364
|
const applyHeadEdits = (html, edits) => {
|
|
9136
9365
|
let out = html;
|
|
9137
9366
|
for (const edit of edits) if (edit.kind === "meta") out = setMeta(out, edit.name, edit.value);
|
|
9367
|
+
else if (edit.kind === "addMeta") out = addMeta(out, edit.name, edit.value);
|
|
9138
9368
|
else if (edit.kind === "addLink") out = addLink(out, edit.rel, edit.href);
|
|
9139
9369
|
else out = removeLink(out, edit.rel, edit.href);
|
|
9140
9370
|
return out;
|
|
@@ -9179,12 +9409,43 @@ const stampFile = (env, path, edits) => Effect.gen(function* () {
|
|
|
9179
9409
|
* `R100` similarity score. Rename similarity is computed tree-to-tree, so a head stamp in the same
|
|
9180
9410
|
* commit lowers it (measured R059-R087 on real memory files). `originalPathFor` is the authoritative
|
|
9181
9411
|
* inverse of the archive mapping, and no correctness path here reads the score.
|
|
9412
|
+
*
|
|
9413
|
+
* **The destination is PROBED, and a taken one gets an ordinal.** `git mv` onto a path that already
|
|
9414
|
+
* holds a file exits 128 (`fatal: destination exists`, measured 2026-08-19), which fails the whole
|
|
9415
|
+
* phase rather than the one file. That is reachable — not hypothetically — because one live path can be
|
|
9416
|
+
* archived TWICE INSIDE ONE YEAR, and the year is the only partition the mapping has:
|
|
9417
|
+
*
|
|
9418
|
+
* - A DETECTED task's path is deliberately deterministic (`tasks.ts`: the digest keys the finding, so a
|
|
9419
|
+
* finding restated tomorrow lands on the same path on purpose). Mint, sweep-close, the finding
|
|
9420
|
+
* reappears, sweep-close again — and the second close aims at the first close's archive path.
|
|
9421
|
+
* - Any path a human restores out of the archive and lets a later night evict again is the same shape
|
|
9422
|
+
* with no detector involved.
|
|
9423
|
+
*
|
|
9424
|
+
* So the fix is here rather than in `tasks.ts`: the collision is a property of the ARCHIVE mapping, and
|
|
9425
|
+
* every phase that archives — retention triage, reprieve, dedup's drops, compress's members — has the
|
|
9426
|
+
* same exposure. The ordinal is `-2`, `-3`, … at the first FREE candidate, deterministic given the tree,
|
|
9427
|
+
* and it goes through `withCollisionOrdinal` so it lands inside `SLUG_MAX_LENGTH` exactly as
|
|
9428
|
+
* `trace-consolidation`'s `freePath` and the store's own collision loop do.
|
|
9429
|
+
*
|
|
9430
|
+
* **A suffixed destination is no longer `originalPathFor`'s inverse, and that is the stated cost.** The
|
|
9431
|
+
* first archiving of a path keeps the unsuffixed name, so `integrity`'s dangling-href repair still finds
|
|
9432
|
+
* it and every existing inverse assertion holds; a SECOND archiving of one path is a second file whose
|
|
9433
|
+
* name says so. Nothing reads the inverse to decide a write — `integrity` searches a known-path set and
|
|
9434
|
+
* falls back to dropping the edge — so the alternative (failing the phase) is strictly worse.
|
|
9435
|
+
*
|
|
9436
|
+
* Exhausting the ordinals logs and answers `null`, which leaves the file at its live path. `null`
|
|
9437
|
+
* already means "not archived" to every caller, so an exhausted probe is counted as a file that did not
|
|
9438
|
+
* move rather than one that was destroyed.
|
|
9182
9439
|
*/
|
|
9183
9440
|
const archiveFile = (env, path, extraEdits = []) => Effect.gen(function* () {
|
|
9184
9441
|
const normalized = normalizePath(path);
|
|
9185
|
-
const target = archivePathFor(normalized, yearOf(env.date));
|
|
9186
9442
|
const html = yield* readFileBytes(env, normalized);
|
|
9187
9443
|
if (html === void 0) return null;
|
|
9444
|
+
const target = yield* freeArchivePath(env, normalized);
|
|
9445
|
+
if (target === void 0) {
|
|
9446
|
+
yield* Effect.logWarning(`sleep.archive refused ${normalized}: every archive ordinal is taken, so the file stays live`);
|
|
9447
|
+
return null;
|
|
9448
|
+
}
|
|
9188
9449
|
yield* attemptIo(`sleep.archive.mkdir:${target}`, async () => {
|
|
9189
9450
|
const { mkdir } = await import("node:fs/promises");
|
|
9190
9451
|
const { dirname } = await import("node:path");
|
|
@@ -9201,6 +9462,43 @@ const archiveFile = (env, path, extraEdits = []) => Effect.gen(function* () {
|
|
|
9201
9462
|
yield* env.deps.git.add([target]);
|
|
9202
9463
|
return target;
|
|
9203
9464
|
});
|
|
9465
|
+
/**
|
|
9466
|
+
* Archive ordinals tried before a file is left where it is. The store's own ceiling and
|
|
9467
|
+
* `trace-consolidation`'s, verbatim, so the three collision loops in this repo agree on the number.
|
|
9468
|
+
*/
|
|
9469
|
+
const ARCHIVE_ORDINAL_LIMIT = 1e3;
|
|
9470
|
+
/**
|
|
9471
|
+
* The lowest free archive path for a live path, or `undefined` when every ordinal is taken.
|
|
9472
|
+
*
|
|
9473
|
+
* DISK is the only authority, and it is enough: a phase archiving one path twice in one run cannot
|
|
9474
|
+
* happen, because the first `git mv` takes the file away and `archiveFile`'s own missing-source read
|
|
9475
|
+
* answers `null` for the second. So there is no in-run `claimed` set to union in, which is the one
|
|
9476
|
+
* half `trace-consolidation`'s `freePath` needs and this does not.
|
|
9477
|
+
*/
|
|
9478
|
+
const freeArchivePath = (env, normalized) => Effect.gen(function* () {
|
|
9479
|
+
const year = yearOf(env.date);
|
|
9480
|
+
for (let ordinal = 1; ordinal <= ARCHIVE_ORDINAL_LIMIT; ordinal += 1) {
|
|
9481
|
+
const candidate = withArchiveOrdinal(archivePathFor(normalized, year), ordinal);
|
|
9482
|
+
if ((yield* readFileBytes(env, candidate)) === void 0) return candidate;
|
|
9483
|
+
}
|
|
9484
|
+
});
|
|
9485
|
+
/**
|
|
9486
|
+
* A path with a collision ordinal spliced into its filename STEM, before the extension.
|
|
9487
|
+
*
|
|
9488
|
+
* Before the extension rather than after, so the result is still an `.html` path the indexer reads and
|
|
9489
|
+
* the parser accepts. Ordinal 1 is the bare path, matching `withCollisionOrdinal`'s own convention, so
|
|
9490
|
+
* the ordinary single-archiving case produces byte-identical paths to what this function replaced.
|
|
9491
|
+
*/
|
|
9492
|
+
const withArchiveOrdinal = (path, ordinal) => {
|
|
9493
|
+
if (ordinal <= 1) return path;
|
|
9494
|
+
const cut = path.lastIndexOf("/");
|
|
9495
|
+
const directory = path.slice(0, cut + 1);
|
|
9496
|
+
const filename = path.slice(cut + 1);
|
|
9497
|
+
const dot = filename.lastIndexOf(".");
|
|
9498
|
+
const stem = dot <= 0 ? filename : filename.slice(0, dot);
|
|
9499
|
+
const extension = dot <= 0 ? "" : filename.slice(dot);
|
|
9500
|
+
return `${directory}${withCollisionOrdinal(stem, ordinal)}${extension}`;
|
|
9501
|
+
};
|
|
9204
9502
|
/** The calendar year an archive path partitions under, from the run's own injected date. */
|
|
9205
9503
|
const yearOf = (date) => {
|
|
9206
9504
|
const year = Number(date.slice(0, 4));
|
|
@@ -9230,7 +9528,16 @@ const datePlusDays = (date, days) => {
|
|
|
9230
9528
|
//#endregion
|
|
9231
9529
|
//#region packages/sleep/dist/env.js
|
|
9232
9530
|
/**
|
|
9233
|
-
* Model assignments per LLM phase
|
|
9531
|
+
* Model assignments per LLM phase.
|
|
9532
|
+
*
|
|
9533
|
+
* Every structured phase names `gpt-5.6-sol`, and the reason is one wire property rather
|
|
9534
|
+
* than a model-quality judgment: its strict `json_schema` mode does constrained decoding
|
|
9535
|
+
* on Bedrock today, so an off-schema answer — including the double-encoded-string shape
|
|
9536
|
+
* that skipped 13 batches in one Claude-5 run (issue #53) — cannot be generated at all.
|
|
9537
|
+
* The Claude 5 models reject `strict` and `output_config.format` on every Bedrock surface
|
|
9538
|
+
* (probed live 2026-08-22), so with them the schema is a request the decode enforces
|
|
9539
|
+
* after the fact, and a violated batch is work lost. When Claude 5 structured outputs
|
|
9540
|
+
* land on Bedrock, re-deciding this map is a quality question again; today it is not.
|
|
9234
9541
|
*
|
|
9235
9542
|
* `trace-consolidation` names `opus-5` and does not thereby choose it. The consolidator is an eve
|
|
9236
9543
|
* agent that pins its own model in `apps/consolidator/agent/agent.ts`, and this map cannot reach that
|
|
@@ -9239,25 +9546,13 @@ const datePlusDays = (date, days) => {
|
|
|
9239
9546
|
* the Bedrock global endpoint, high reasoning effort, no cost ceiling.)
|
|
9240
9547
|
*/
|
|
9241
9548
|
const DEFAULT_MODELS = {
|
|
9242
|
-
|
|
9243
|
-
|
|
9244
|
-
|
|
9245
|
-
|
|
9246
|
-
|
|
9247
|
-
|
|
9248
|
-
|
|
9249
|
-
"dedup-merge": "sonnet-5",
|
|
9250
|
-
/**
|
|
9251
|
-
* Sonnet, and one or two calls a night: the whole of one entity type's name list goes in one call.
|
|
9252
|
-
* The question is a partition over short strings with their evidence inline, not a synthesis, so
|
|
9253
|
-
* the strong model would buy nothing the deterministic floors around the answer do not already
|
|
9254
|
-
* supply.
|
|
9255
|
-
*/
|
|
9256
|
-
"entity-resolution": "sonnet-5",
|
|
9257
|
-
"edge-typing": "sonnet-5",
|
|
9258
|
-
"arc-synthesis": "opus-5",
|
|
9259
|
-
compress: "sonnet-5",
|
|
9260
|
-
"trace-consolidation": "opus-5"
|
|
9549
|
+
"dedup-merge": "gpt-5.6-sol",
|
|
9550
|
+
"entity-resolution": "gpt-5.6-sol",
|
|
9551
|
+
"edge-typing": "gpt-5.6-sol",
|
|
9552
|
+
"arc-synthesis": "gpt-5.6-sol",
|
|
9553
|
+
compress: "gpt-5.6-sol",
|
|
9554
|
+
"trace-consolidation": "opus-5",
|
|
9555
|
+
"task-detection": "gpt-5.6-sol"
|
|
9261
9556
|
};
|
|
9262
9557
|
/** The model a phase calls: the caller's override, else {@link DEFAULT_MODELS}, else sonnet. */
|
|
9263
9558
|
const modelFor = (deps, phase) => deps.models?.[phase] ?? DEFAULT_MODELS[phase] ?? "sonnet-5";
|
|
@@ -9615,6 +9910,94 @@ than once, in different words.
|
|
|
9615
9910
|
own dates afterwards, and a proposed group is still checked for contradicting claims, differing
|
|
9616
9911
|
numbers, and differing product variants before anything is written. Answer only the question of
|
|
9617
9912
|
sameness.`;
|
|
9913
|
+
/**
|
|
9914
|
+
* What a detected finding is: a commitment somebody made, or a follow-up nobody closed.
|
|
9915
|
+
*
|
|
9916
|
+
* Two values and not more. Issue #44 names both — "an open commitment or unresolved follow-up" — and
|
|
9917
|
+
* they are genuinely different work: a commitment has an actor who said they would do something, and a
|
|
9918
|
+
* follow-up is a question or a defect the text leaves open with nobody attached. The phase renders a
|
|
9919
|
+
* different claim for each, and the pair is closed because a third value would be a category whose
|
|
9920
|
+
* reading nothing downstream could state.
|
|
9921
|
+
*/
|
|
9922
|
+
const TaskFindingKind = Schema.Literals(["commitment", "followup"]);
|
|
9923
|
+
/** One finding about one offered member. */
|
|
9924
|
+
const TaskFinding = Schema.Struct({
|
|
9925
|
+
/** The offered key, e.g. `m3`. A key the batch never held resolves to nothing and is dropped. */
|
|
9926
|
+
memberKey: Schema.String,
|
|
9927
|
+
/**
|
|
9928
|
+
* The sentence that carries the finding, copied VERBATIM from the member's text.
|
|
9929
|
+
*
|
|
9930
|
+
* Verbatim is a checked requirement and not a request: the phase looks the sentence up in the cited
|
|
9931
|
+
* file's own article text and refuses the mint when it is not there. So a paraphrase costs the
|
|
9932
|
+
* finding, which is why the system prompt says so in those words.
|
|
9933
|
+
*/
|
|
9934
|
+
sentence: Schema.String,
|
|
9935
|
+
kind: TaskFindingKind,
|
|
9936
|
+
/** Unitless in `[0, 1]`. The mint gate is deterministic and reads this, not the prose. */
|
|
9937
|
+
confidence: Schema.Finite.check(Schema.isBetween({
|
|
9938
|
+
minimum: 0,
|
|
9939
|
+
maximum: 1
|
|
9940
|
+
}))
|
|
9941
|
+
});
|
|
9942
|
+
/**
|
|
9943
|
+
* One batch's whole answer: the findings across every member it was shown.
|
|
9944
|
+
*
|
|
9945
|
+
* `findings: []` is a refusal and the correct answer for most batches. Most memories record a fact and
|
|
9946
|
+
* carry no open work at all, and a model that felt obliged to fill the list would mint tasks out of
|
|
9947
|
+
* ordinary prose — which is precisely the noise the volume cap exists to bound and the reviewer's
|
|
9948
|
+
* attention cannot absorb.
|
|
9949
|
+
*/
|
|
9950
|
+
const TaskDetection = Schema.Struct({ findings: Schema.Array(TaskFinding) });
|
|
9951
|
+
/**
|
|
9952
|
+
* The task-detection system prompt.
|
|
9953
|
+
*
|
|
9954
|
+
* The conservative posture every other judge in this file carries, aimed at the one thing this phase
|
|
9955
|
+
* can get wrong at scale: a memory that MENTIONS work is not a memory that carries an open
|
|
9956
|
+
* commitment. An `error_pattern` describing a defect somebody already fixed reads exactly like one
|
|
9957
|
+
* describing a defect nobody has, and the difference is in whether the text says it was resolved.
|
|
9958
|
+
*
|
|
9959
|
+
* The verbatim rule is stated as a consequence rather than as a style note, because it IS one: the
|
|
9960
|
+
* phase looks the sentence up in the file and drops the finding when it is absent.
|
|
9961
|
+
*/
|
|
9962
|
+
const TASK_DETECT_SYSTEM = `You find OPEN WORK recorded in an AI agent's long-term memory system. You are given a NUMBERED LIST
|
|
9963
|
+
of memories. For each one, decide whether its text records work that is still open, and if so quote
|
|
9964
|
+
the sentence that says so.
|
|
9965
|
+
|
|
9966
|
+
Two kinds:
|
|
9967
|
+
|
|
9968
|
+
- commitment: somebody stated they would do something and the text does not say it happened. "I'll
|
|
9969
|
+
fix that tomorrow", "we need to wire capture before the next release", "leaving the merge until you
|
|
9970
|
+
review it".
|
|
9971
|
+
- followup: the text leaves something unresolved with nobody attached. An unfixed defect a memory
|
|
9972
|
+
describes, a question it ends on, a decision it says is blocked pending something else.
|
|
9973
|
+
|
|
9974
|
+
Rules:
|
|
9975
|
+
|
|
9976
|
+
- sentence must be copied VERBATIM from the member's own text, character for character. The system
|
|
9977
|
+
looks it up in the file and DISCARDS the finding when it is not found, so a paraphrase, a
|
|
9978
|
+
correction, a stitched-together sentence, or a summary loses the finding entirely.
|
|
9979
|
+
- Return findings: [] when nothing here carries open work. That is the ordinary answer: most memories
|
|
9980
|
+
record a fact, not a task. Refusing is correct whenever you are unsure.
|
|
9981
|
+
- A memory that DESCRIBES completed work is not open work. "we fixed the flaky teardown by pinning
|
|
9982
|
+
the port" is a record, not a task. Look for work the text leaves undone.
|
|
9983
|
+
- Never report a hypothetical, an option considered and rejected, or a general principle. "if the
|
|
9984
|
+
cache misses we would need to warm it" names no work anybody owes.
|
|
9985
|
+
- One finding per memory at most, and only for the memories that have one. Omitting a member is
|
|
9986
|
+
always allowed.
|
|
9987
|
+
- Rate confidence honestly. A finding above the floor becomes a task file a human is asked to review,
|
|
9988
|
+
and a queue full of things that were never work is a queue nobody reads.`;
|
|
9989
|
+
/** The instruction that closes a task-detection batch's user turn, after the member list. */
|
|
9990
|
+
const TASK_DETECT_INSTRUCTION = "Which of these memories carry open work? For each one that does, name it by its offered key, quote the sentence verbatim, say whether it is a commitment or a followup, and rate your confidence. Return findings: [] if none of them do.";
|
|
9991
|
+
/**
|
|
9992
|
+
* The task-detection user turn for one batch: every member's text under its offered key.
|
|
9993
|
+
*
|
|
9994
|
+
* `batchPrompt` from the kernel builds the list and appends the instruction, so
|
|
9995
|
+
* {@link TASK_DETECT_SYSTEM} plus the tool schema form the cache-eligible prefix and only the member
|
|
9996
|
+
* list is new bytes per batch. The label is `memory` rather than `member`, because what the model is
|
|
9997
|
+
* asked about is whether a MEMORY records open work, and the wrapper's label is the only place the
|
|
9998
|
+
* prompt names the thing.
|
|
9999
|
+
*/
|
|
10000
|
+
const taskDetectPrompt = (members) => batchPrompt(members, TASK_DETECT_INSTRUCTION, { label: "memory" });
|
|
9618
10001
|
/** One labelled corpus block, delimited so its prose cannot be read as an instruction. */
|
|
9619
10002
|
const dataBlock = (label, text) => wrapAsData(label, text);
|
|
9620
10003
|
/**
|
|
@@ -9698,7 +10081,7 @@ const dedupPrompt = (components) => {
|
|
|
9698
10081
|
/**
|
|
9699
10082
|
* The memory type no phase of a sleep cycle touches.
|
|
9700
10083
|
*
|
|
9701
|
-
* A task is live working state, and every one of the
|
|
10084
|
+
* A task is live working state, and every one of the sixteen phases is a judgment about REMEMBERED
|
|
9702
10085
|
* FACTS: decay says a claim is fading, dedup says two claims are one, edge typing says one claim
|
|
9703
10086
|
* caused or contradicts another, retention says a claim has stopped earning its place. None of those hold for
|
|
9704
10087
|
* a thing an agent intends to do, and each would be wrong applied to one. A task the agent has
|
|
@@ -9725,6 +10108,33 @@ const isSleepExcluded = (memoryType) => SLEEP_EXCLUDED_TYPES.includes(memoryType
|
|
|
9725
10108
|
const activeCorpus = (db) => db.all(`SELECT path, memory_type, title, gist, body_text, content_hash, confidence, importance,
|
|
9726
10109
|
word_count, created_at, updated_at, valid_until, reprieves
|
|
9727
10110
|
FROM files WHERE archived = 0 ORDER BY created_at ASC, path ASC`);
|
|
10111
|
+
/**
|
|
10112
|
+
* The most recently touched active NON-TASK memories, newest first, capped.
|
|
10113
|
+
*
|
|
10114
|
+
* Task detection's candidate slice. Issue #44 asks for "recent/high-salience", and this is the RECENT
|
|
10115
|
+
* half alone, which is the deliberate cut. The salience half would mean a retention pass — label
|
|
10116
|
+
* propagation plus PageRank over the whole edge list plus the access plane — for a scan whose job is
|
|
10117
|
+
* to notice text nobody has resolved yet, and salience measures the opposite: how much a memory has
|
|
10118
|
+
* been leaned on since it was written. A commitment made last night has no access history at all, so
|
|
10119
|
+
* ranking by salience would systematically rank the phase's best candidates last.
|
|
10120
|
+
*
|
|
10121
|
+
* `updated_at DESC` and not `created_at`, because a memory CORRECTED yesterday carries yesterday's
|
|
10122
|
+
* text, which is the text a commitment would be in. `path ASC` breaks the tie, so the slice — and
|
|
10123
|
+
* therefore the batch boundaries and the `m1`..`mN` keys — is a function of the corpus rather than of
|
|
10124
|
+
* the order rows came back in.
|
|
10125
|
+
*
|
|
10126
|
+
* Tasks are excluded here, in the statement, matching every other read in this module. That is the
|
|
10127
|
+
* no-self-referential-loop guard issue #44 names, and putting it in SQL rather than in the phase means
|
|
10128
|
+
* a detected task cannot become evidence of another task even if a caller forgot to filter. The phase
|
|
10129
|
+
* carries a second, path-level check for the same invariant, because this one is keyed on a projected
|
|
10130
|
+
* column and the projection is refreshed once per night.
|
|
10131
|
+
*/
|
|
10132
|
+
const recentActiveMemories = (db, options) => db.all(`SELECT path, memory_type, title, gist, body_text, content_hash, confidence, importance,
|
|
10133
|
+
word_count, created_at, updated_at, valid_until, reprieves
|
|
10134
|
+
FROM files
|
|
10135
|
+
WHERE archived = 0 AND memory_type NOT IN (${typePlaceholders()})
|
|
10136
|
+
ORDER BY updated_at DESC, path ASC
|
|
10137
|
+
LIMIT ?`, [...SLEEP_EXCLUDED_TYPES, options.limit]);
|
|
9728
10138
|
/** A `memory_type NOT IN (…)` clause against alias `f`, or nothing when nothing is excluded. */
|
|
9729
10139
|
const typeFilterFor = (alias, excluded) => excluded.length === 0 ? "" : ` AND ${alias}.memory_type NOT IN (${excluded.map(() => "?").join(", ")})`;
|
|
9730
10140
|
/**
|
|
@@ -10803,6 +11213,609 @@ const confidenceDecay = (env) => Effect.gen(function* () {
|
|
|
10803
11213
|
};
|
|
10804
11214
|
});
|
|
10805
11215
|
|
|
11216
|
+
//#endregion
|
|
11217
|
+
//#region packages/sleep/dist/tasks.js
|
|
11218
|
+
/**
|
|
11219
|
+
* The minting discipline for a DETECTED task: one home for idempotence, evidence verification, the
|
|
11220
|
+
* nightly cap, and self-cleaning.
|
|
11221
|
+
*
|
|
11222
|
+
* Issue #44's shape is three detection surfaces sharing one discipline. The surfaces differ in what
|
|
11223
|
+
* they notice — a review band entity-resolution declined to merge, a near-duplicate pair the
|
|
11224
|
+
* divergence veto refused, a commitment a model found in a memory's own prose — and they agree on
|
|
11225
|
+
* everything that happens after: the finding becomes a `task` file authored `agent:sleep`, keyed on a
|
|
11226
|
+
* stable digest so a second night refreshes rather than duplicates, carrying its evidence verbatim,
|
|
11227
|
+
* inside a nightly volume cap, and closed when the finding stops appearing. That agreement is what
|
|
11228
|
+
* lives here. A copy of it per surface would be three chances to mint a task nobody can trust.
|
|
11229
|
+
*
|
|
11230
|
+
* ## A detected task is a `task`, and inherits every firewall by being one
|
|
11231
|
+
*
|
|
11232
|
+
* Nothing below teaches sleep about a new kind of file. `task` is already a memory type with its own
|
|
11233
|
+
* lifecycle (`todo/doing/blocked/done`, and `done` archives), its own edge class, its own placement
|
|
11234
|
+
* rule, and — the part that matters here — a standing exclusion from every sleep phase
|
|
11235
|
+
* (`SLEEP_EXCLUDED_TYPES`), from retrieval by default, from the salience arm, and from the
|
|
11236
|
+
* `files_content_hash_active` dedup index. So a file this module writes is invisible to dedup,
|
|
11237
|
+
* compress, retention, edge typing, entity resolution, and person links from the moment it lands,
|
|
11238
|
+
* with no phase needing to learn about it. That is the whole reason detection mints a task instead of
|
|
11239
|
+
* a new artifact class.
|
|
11240
|
+
*
|
|
11241
|
+
* ## The KEY is the path, not a meta and not the content hash
|
|
11242
|
+
*
|
|
11243
|
+
* A detected task's idempotence surface is its PATH: `areas/inbox/tasks/det-<12 hex>-<slug>.html`,
|
|
11244
|
+
* where the digest is {@link detectionKey} over the detector's name and a canonical finding string.
|
|
11245
|
+
* Three properties follow, and each is why the alternatives were declined:
|
|
11246
|
+
*
|
|
11247
|
+
* - **Not the content hash.** `files_content_hash_active` deliberately carves out open tasks — two
|
|
11248
|
+
* open tasks with identical bodies are two real work items — so the structural dedup key cannot
|
|
11249
|
+
* answer "have I already minted this finding". A detection that relied on it would mint a second
|
|
11250
|
+
* task every night and the index would admit every one of them.
|
|
11251
|
+
* - **Not a new `memhtml-detection` meta.** The meta vocabulary is CLOSED and ordered
|
|
11252
|
+
* (`packages/html/src/vocabulary.ts`), so a new name is a format change plus a parse change plus a
|
|
11253
|
+
* projection plus a migration plus an index, and the lookup it would buy is a lookup the path
|
|
11254
|
+
* already answers. A path also survives `rm index.db && rebuild` with no projection at all, which a
|
|
11255
|
+
* queryable column does not.
|
|
11256
|
+
* - **Collision-free by construction, at the LIVE path.** Every stem begins with a distinct digest, so
|
|
11257
|
+
* two different findings cannot land on one live path however their titles slug. This module therefore
|
|
11258
|
+
* needs none of the ordinal-suffix search `trace-consolidation`'s `freePath` performs, and cannot
|
|
11259
|
+
* silently overwrite a file the way that probe exists to prevent: a live path that is already taken is
|
|
11260
|
+
* BY DEFINITION the same finding, which is the refresh case rather than a collision.
|
|
11261
|
+
*
|
|
11262
|
+
* The ARCHIVE path is the one place that reasoning inverts, and `archiveFile` owns it. Determinism is
|
|
11263
|
+
* what makes a mint idempotent and is exactly what makes two archivings of one key collide, because
|
|
11264
|
+
* `archivePathFor` partitions only by YEAR: mint → sweep-close → the finding reappears → sweep-close
|
|
11265
|
+
* again lands both closures on one archive path inside one year, and `git mv` exits 128 on a
|
|
11266
|
+
* destination that exists. So the ordinal search lives in `edits.ts`, where every archiving phase gets
|
|
11267
|
+
* it, and the dismissal scan below is a PREFIX scan rather than a derived path because of it.
|
|
11268
|
+
*
|
|
11269
|
+
* ## The TREE is read, never the index
|
|
11270
|
+
*
|
|
11271
|
+
* Every lookup here is a `readdir` plus a file read under {@link DETECTED_TASK_DIR}. The index is
|
|
11272
|
+
* refreshed once, in preflight, and not again, so a task minted earlier in the same night is absent
|
|
11273
|
+
* from it — and two detectors reaching one finding in one night is exactly the case idempotence has
|
|
11274
|
+
* to cover. Reading the tree makes a mint see the mints before it. The cost is bounded by
|
|
11275
|
+
* {@link DETECTED_TASK_CAP} reads of one directory per phase, because self-cleaning keeps that
|
|
11276
|
+
* directory the size of the OPEN detected queue rather than of the corpus.
|
|
11277
|
+
*
|
|
11278
|
+
* ## A human's closure is a STANDING DISMISSAL, and it is durable
|
|
11279
|
+
*
|
|
11280
|
+
* A detected task a human closed is a proposal they ANSWERED. `done` archives, so the file leaves the
|
|
11281
|
+
* open queue — and the open queue was the only thing a mint used to consult, so the same finding minted
|
|
11282
|
+
* a fresh task the next night, and the night after, forever. There was no way to say no. That is worse
|
|
11283
|
+
* than noise: a queue whose items cannot be dismissed is a queue a human stops reading, which is the
|
|
11284
|
+
* failure {@link DETECTED_TASK_CAP} and the sweep both exist to prevent from the other direction.
|
|
11285
|
+
*
|
|
11286
|
+
* So {@link mintDetectedTask} also reads the ARCHIVE, and a `done`-stamped detected task under the same
|
|
11287
|
+
* key with NO {@link MACHINE_CLOSED_TAG} is a standing dismissal: the mint is declined and counted
|
|
11288
|
+
* `dismissed`. Three properties of the shape:
|
|
11289
|
+
*
|
|
11290
|
+
* - **The discriminator is a tag on the file, because nothing else told the two closures apart.** See
|
|
11291
|
+
* {@link MACHINE_CLOSED_TAG}. A SWEPT task must not read as a dismissal — the finding vanishing and a
|
|
11292
|
+
* human declining it are opposite facts, and a finding that reappears after vanishing is new
|
|
11293
|
+
* information a reviewer should see.
|
|
11294
|
+
* - **It is a PREFIX SCAN of the year buckets, not a derived path.** `archiveFile`'s ordinal suffixing
|
|
11295
|
+
* means one key can own `det-<key>-<slug>.html` and `det-<key>-<slug>-2.html` in one year, so there is
|
|
11296
|
+
* no single path to probe. The scan is a `readdir` per year over
|
|
11297
|
+
* `archive/<YYYY>/areas/inbox/tasks/`, the same TREE read `openDetections` performs, bounded by
|
|
11298
|
+
* {@link DISMISSAL_LOOKBACK_YEARS}.
|
|
11299
|
+
* - **Dismissal is durable within the archive's VISIBILITY.** A human who wants the review back deletes
|
|
11300
|
+
* the archived file (or moves it out of the year buckets), and the next night mints it again. Nothing
|
|
11301
|
+
* else undoes it, which is the point: a dismissal that expired on a timer would be a queue item that
|
|
11302
|
+
* came back for no reason a human could name.
|
|
11303
|
+
*
|
|
11304
|
+
* ## What the code verifies, and what it cannot
|
|
11305
|
+
*
|
|
11306
|
+
* A quote is checked against the cited file's own article text and a mint whose quote is not there is
|
|
11307
|
+
* REFUSED ({@link mintDetectedTask} answers `unverified`). That is the issue's "proposal with
|
|
11308
|
+
* evidence, never an assertion", enforced rather than asked for: the one detector whose evidence a
|
|
11309
|
+
* model supplies is surface 3, and a fabricated sentence must not reach a file a human then reads as
|
|
11310
|
+
* a citation.
|
|
11311
|
+
*
|
|
11312
|
+
* A MEASUREMENT is a different thing and {@link DetectionEvidence} says so in the type. The evidence
|
|
11313
|
+
* behind an entity-resolution review candidate is a character-similarity ratio and two file counts —
|
|
11314
|
+
* a fact about the corpus that no sentence anywhere states — so there is nothing to verify it
|
|
11315
|
+
* against, and pretending otherwise by quoting an arbitrary claiming memory would manufacture a
|
|
11316
|
+
* citation to satisfy a check. The union keeps the difference visible at every call site instead of
|
|
11317
|
+
* leaving it to convention.
|
|
11318
|
+
*
|
|
11319
|
+
* A SESSION is the third, and it is the one place issue #44's "body must quote its source verbatim"
|
|
11320
|
+
* is deliberately NOT satisfied, because a stronger invariant refuses it. Surface 2's evidence is a
|
|
11321
|
+
* transcript line, and `.memhtml` holds no session content: the trace plane is a read-only index over
|
|
11322
|
+
* `~/.claude/projects`, and `phases/trace-consolidation.ts` states — with a byte-level test behind it —
|
|
11323
|
+
* that a distilled claim reaches the corpus and its verbatim quote does not. A quote copied into a
|
|
11324
|
+
* task body would be exactly the leak that test exists to catch. So the `session` arm carries the
|
|
11325
|
+
* session ID and no quote: the ID becomes a `memhtml-session` stamp (a projected column, so
|
|
11326
|
+
* `files.session_id` answers "which session opened this"), the body names the session as the place to
|
|
11327
|
+
* look, and the verbatim line goes where every other trace-consolidation quote goes, into the COMMIT
|
|
11328
|
+
* MESSAGE, which is not indexed, not chunked, not embedded, and not retrievable.
|
|
11329
|
+
*
|
|
11330
|
+
* What verifies a `session` quote is therefore not code in this file but the client boundary:
|
|
11331
|
+
* `ungroundedCommitmentReason` (`apps/consolidator/src/contract.ts`) refuses the whole turn when a
|
|
11332
|
+
* commitment cites a session the run did not make readable, and the phase additionally drops a
|
|
11333
|
+
* commitment whose session is outside the batch it asked about. Re-verifying the quote against
|
|
11334
|
+
* transcript bytes here was considered and declined: it would require `MEMHTML_TRACE_ROOT` in
|
|
11335
|
+
* `PhaseEnv`, which `consolidator.ts` records as the thing deliberately kept out of the environment
|
|
11336
|
+
* all sixteen phases share, and it would buy a check against a file that may have rotated away since
|
|
11337
|
+
* the consolidator read it.
|
|
11338
|
+
*/
|
|
11339
|
+
/**
|
|
11340
|
+
* Where a detected task lands: the ordinary task placement, asked of `@memhtml/contracts` rather than
|
|
11341
|
+
* retyped.
|
|
11342
|
+
*
|
|
11343
|
+
* A detected task is an ordinary task and files where one files. `placementFor` routes a workspaceless
|
|
11344
|
+
* task to `areas/inbox/tasks`, and `memhtml doctor` reports inbox depth, so a noisy detector shows up
|
|
11345
|
+
* as a health signal on the surface built to carry one. A parallel `detected/` tree would be a second
|
|
11346
|
+
* place to look for the same work and would sit outside the four PARA buckets the indexer reads.
|
|
11347
|
+
*/
|
|
11348
|
+
const DETECTED_TASK_DIR = placementFor({ memoryType: "task" });
|
|
11349
|
+
/** The filename prefix that makes a detected task recognizable from its path alone. */
|
|
11350
|
+
const DETECTION_PREFIX = "det-";
|
|
11351
|
+
/**
|
|
11352
|
+
* Hex characters of the digest carried in a path.
|
|
11353
|
+
*
|
|
11354
|
+
* Twelve, which is git's own abbreviated-sha width and the same width the report renderer prints. It
|
|
11355
|
+
* leaves 48 bits against a queue whose size the cap and the sweep hold in the tens, and it costs 17
|
|
11356
|
+
* characters of the 80-character slug budget rather than 68.
|
|
11357
|
+
*/
|
|
11358
|
+
const DETECTION_DIGEST_CHARS = 12;
|
|
11359
|
+
/** The tag every detected task carries first, so `task list` can filter the machine's queue. */
|
|
11360
|
+
const DETECTED_TAG = "detected";
|
|
11361
|
+
/**
|
|
11362
|
+
* The tag a MACHINE closure appends, which is the only thing that tells one apart from a human's.
|
|
11363
|
+
*
|
|
11364
|
+
* **It had to be added, because nothing in a file distinguished the two.** Both closure paths write
|
|
11365
|
+
* exactly `memhtml-task-status: done` plus the three archive stamps — `closeVanishedDetections` here and
|
|
11366
|
+
* `memhtml task status done` in `apps/cli/src/operations.ts`, deliberately, so a detected task closes the
|
|
11367
|
+
* same way a human's does. The differing REASON ("no longer detected" against "task done") lives only in
|
|
11368
|
+
* the commit message, and a commit message is not a queryable fact about a file: reading it would mean a
|
|
11369
|
+
* `git log --follow` per archived path per mint, and it says nothing at all after a rebase or a
|
|
11370
|
+
* `git filter-repo`. So the discriminator is a `memhtml-tag`, which is already repeatable, already in the
|
|
11371
|
+
* closed vocabulary, already projects to `file_tags`, and travels with the file's bytes.
|
|
11372
|
+
*
|
|
11373
|
+
* The value of the distinction is {@link mintDetectedTask}'s dismissal check. A HUMAN closing a detected
|
|
11374
|
+
* task is answering the proposal — "I looked, and I do not want this review" — and re-minting it the next
|
|
11375
|
+
* night makes the queue un-dismissable. A MACHINE closing it means the finding stopped appearing or the
|
|
11376
|
+
* system resolved it, and a finding that comes back later is genuinely new information. Without the tag
|
|
11377
|
+
* the two are one archived `done` task, and the check would have to pick one reading and be wrong about
|
|
11378
|
+
* the other half.
|
|
11379
|
+
*
|
|
11380
|
+
* Appended, never set, so it lands AFTER the detector tag and {@link openDetections}' positional read of
|
|
11381
|
+
* `tags[0]`/`tags[1]` is untouched.
|
|
11382
|
+
*/
|
|
11383
|
+
const MACHINE_CLOSED_TAG = "machine-closed";
|
|
11384
|
+
/**
|
|
11385
|
+
* Archive year partitions searched for a standing dismissal.
|
|
11386
|
+
*
|
|
11387
|
+
* Ten, the same span `integrity`'s `ARCHIVE_LOOKBACK_YEARS` chases a dangling href over, and stated here
|
|
11388
|
+
* rather than imported because a module may not depend on a phase. The number is what bounds a
|
|
11389
|
+
* dismissal's DURABILITY: a task dismissed eleven years ago can be re-minted, which is the honest
|
|
11390
|
+
* reading of "durable within the archive's visibility" rather than a promise the code cannot keep.
|
|
11391
|
+
*/
|
|
11392
|
+
const DISMISSAL_LOOKBACK_YEARS = 10;
|
|
11393
|
+
/**
|
|
11394
|
+
* Detected tasks one night may mint, across every detector.
|
|
11395
|
+
*
|
|
11396
|
+
* Ten, from issue #44 verbatim: "a noisy detector that mints 200 tasks destroys the working set it
|
|
11397
|
+
* exists to serve". The budget is SHARED rather than per-detector, because the number a human can
|
|
11398
|
+
* review is a property of the human and not of how many detectors sleep happens to run. Overflow is
|
|
11399
|
+
* counted, never silently dropped — a detector pressing against the cap every night is a detector
|
|
11400
|
+
* whose threshold is wrong, and that is only visible in the counts.
|
|
11401
|
+
*
|
|
11402
|
+
* A REFRESH costs nothing. The cap bounds new work arriving in the queue, and re-stamping a task a
|
|
11403
|
+
* human has already been shown adds none.
|
|
11404
|
+
*/
|
|
11405
|
+
const DETECTED_TASK_CAP = 10;
|
|
11406
|
+
/** Characters of a detected task's title. The same 90 a distilled memory's title is cut to. */
|
|
11407
|
+
const TITLE_CHARS = 90;
|
|
11408
|
+
/**
|
|
11409
|
+
* The slug budget left for a title once the prefix and digest are spent.
|
|
11410
|
+
*
|
|
11411
|
+
* Derived, not chosen, so the two cannot drift into a stem that breaches `SLUG_MAX_LENGTH` — which
|
|
11412
|
+
* `isSlug` rejects, and every other path in the corpus satisfies it.
|
|
11413
|
+
*/
|
|
11414
|
+
const STEM_SLUG_CHARS = 80 - 4 - 12 - 1;
|
|
11415
|
+
/**
|
|
11416
|
+
* A finding's stable key: `det-<12 hex>` over the detector's name and a canonical finding string.
|
|
11417
|
+
*
|
|
11418
|
+
* The finding string is the CALLER's canonical form of what it noticed — an entity type plus two
|
|
11419
|
+
* sorted names, two sorted paths, a rel plus two sorted paths — and sorting is the caller's job
|
|
11420
|
+
* because only the caller knows which of its fields are unordered. What this adds is the
|
|
11421
|
+
* normalization every caller would otherwise repeat: NFC, lowercase, collapsed whitespace. So a
|
|
11422
|
+
* finding restated with different spacing on a later night keys the same, and a night that saw
|
|
11423
|
+
* `(a, b)` keys with a night that saw `(b, a)` provided the caller sorted.
|
|
11424
|
+
*
|
|
11425
|
+
* The detector's name is INSIDE the digest, so two detectors that happen to canonicalize one finding
|
|
11426
|
+
* identically still own separate tasks. They noticed different things about it, and a sweep is
|
|
11427
|
+
* per-detector: sharing a key would let one detector's sweep close the other's task.
|
|
11428
|
+
*/
|
|
11429
|
+
const detectionKey = (detector, finding) => `${DETECTION_PREFIX}${createHash("sha256").update(`${normalizeFinding(detector)}${normalizeFinding(finding)}`, "utf8").digest("hex").slice(0, 12)}`;
|
|
11430
|
+
/** NFC, lowercase, collapsed whitespace, trimmed. The pre-digest form. */
|
|
11431
|
+
const normalizeFinding = (text) => text.normalize("NFC").toLowerCase().replace(/\s+/g, " ").trim();
|
|
11432
|
+
/** The path a key and a title name. Total, and unique per key whatever the title slugs to. */
|
|
11433
|
+
const detectedTaskPath = (key, title) => {
|
|
11434
|
+
const stem = slugify(title).slice(0, STEM_SLUG_CHARS).replace(/-+$/, "");
|
|
11435
|
+
return `${DETECTED_TASK_DIR}/${key}-${stem === "" ? SLUG_FALLBACK : stem}.html`;
|
|
11436
|
+
};
|
|
11437
|
+
/** The key a detected task's path carries, or `undefined` when the path is not one. */
|
|
11438
|
+
const detectionKeyOf = (path) => {
|
|
11439
|
+
const filename = path.slice(path.lastIndexOf("/") + 1);
|
|
11440
|
+
return new RegExp(`^(${DETECTION_PREFIX}[0-9a-f]{${String(12)}})-`).exec(filename)?.[1];
|
|
11441
|
+
};
|
|
11442
|
+
/** True when a path is a detected task's. The self-scan guard surface-3 reads. */
|
|
11443
|
+
const isDetectedTaskPath = (path) => detectionKeyOf(path) !== void 0;
|
|
11444
|
+
/** A fresh budget at the nightly cap. */
|
|
11445
|
+
const makeDetectionBudget = (cap = 10) => ({
|
|
11446
|
+
remaining: Math.max(0, Math.trunc(cap)),
|
|
11447
|
+
overflow: 0
|
|
11448
|
+
});
|
|
11449
|
+
/**
|
|
11450
|
+
* The run's shared budget, or a fresh one when the run did not supply it.
|
|
11451
|
+
*
|
|
11452
|
+
* `PhaseEnv.detectionBudget` is optional so the five existing construction sites keep compiling, and
|
|
11453
|
+
* the fallback is what makes a phase driven directly by a test behave like one inside a run: it gets
|
|
11454
|
+
* the full cap to itself. A caller must call this ONCE per phase invocation and thread the result,
|
|
11455
|
+
* because calling it per mint against an absent field would hand out a fresh cap every time.
|
|
11456
|
+
*/
|
|
11457
|
+
const budgetFor = (env) => env.detectionBudget ?? makeDetectionBudget();
|
|
11458
|
+
/**
|
|
11459
|
+
* Mint a detected task, or refresh the one this finding already owns.
|
|
11460
|
+
*
|
|
11461
|
+
* Order matters and is stated once here, because each step's position is what makes it mean
|
|
11462
|
+
* something:
|
|
11463
|
+
*
|
|
11464
|
+
* 1. **The existing open detections are read from the tree.** A key already present is the refresh
|
|
11465
|
+
* case, and refreshing costs no budget.
|
|
11466
|
+
* 2. **The archive is read for a standing dismissal**, and a human's closure of this key declines the
|
|
11467
|
+
* mint. Second rather than first because the open queue is one `readdir` and this is up to
|
|
11468
|
+
* {@link DISMISSAL_LOOKBACK_YEARS} of them, so the common case — the key is open, refresh it — pays
|
|
11469
|
+
* for none of them. See the module header for what makes a dismissal durable.
|
|
11470
|
+
* 3. **The evidence is verified before anything is written.** A quote absent from the file it cites
|
|
11471
|
+
* refuses the whole mint. Checking after the write would leave a task in the tree asserting a
|
|
11472
|
+
* citation the corpus does not support, and a later commit removing it would still be in the log.
|
|
11473
|
+
* 4. **The frame-key check runs against the OPEN queue only.** Two detectors describing one work
|
|
11474
|
+
* item in different words key differently, so the digest cannot catch them; a shared claim slot
|
|
11475
|
+
* can. It fires rarely by construction, because `frameKeyOf`'s guards fail closed on ordinary
|
|
11476
|
+
* prose — which is the honest scope of this check and the reason it is the second net rather than
|
|
11477
|
+
* the first. **Every caller's claim must be frame-DISTINCT per finding**: a claim whose frame key is
|
|
11478
|
+
* a constant makes this check cap that detector's whole queue at one task, which is the defect the
|
|
11479
|
+
* five minters' claim shapes are now measured against.
|
|
11480
|
+
* 5. **The budget is spent last**, so a refusal at any earlier step does not consume a night's
|
|
11481
|
+
* allowance.
|
|
11482
|
+
*
|
|
11483
|
+
* Staging only. The phase that called this commits, so a detected task lands in the SAME commit as
|
|
11484
|
+
* the work that found it, which is what puts it behind the discrimination gate and what makes the
|
|
11485
|
+
* commit reviewable as one decision.
|
|
11486
|
+
*/
|
|
11487
|
+
const mintDetectedTask = (env, budget, request) => Effect.gen(function* () {
|
|
11488
|
+
const key = detectionKey(request.detector, request.finding);
|
|
11489
|
+
const open = yield* openDetections(env);
|
|
11490
|
+
const existing = open.find((detected) => detected.key === key);
|
|
11491
|
+
if (existing !== void 0) {
|
|
11492
|
+
/**
|
|
11493
|
+
* A second night refreshes the stamp and writes nothing else. Not the claim, not the evidence,
|
|
11494
|
+
* not the detail: a human may have edited the body or moved the status to `doing`, and a
|
|
11495
|
+
* detector overwriting that would take the queue away from the person it exists to serve.
|
|
11496
|
+
* `stampFile` returns false when the instant is already stamped, so a same-date re-run is free.
|
|
11497
|
+
*/
|
|
11498
|
+
yield* stampFile(env, existing.path, [meta("memhtml-updated", env.at)]);
|
|
11499
|
+
return "refreshed";
|
|
11500
|
+
}
|
|
11501
|
+
if (yield* humanDismissed(env, key)) {
|
|
11502
|
+
yield* Effect.logInfo(`sleep.tasks ${request.detector} declined a mint: a human closed ${key} and the dismissal stands`);
|
|
11503
|
+
return "dismissed";
|
|
11504
|
+
}
|
|
11505
|
+
if (!(yield* evidenceHolds(env, request.evidence))) {
|
|
11506
|
+
yield* Effect.logWarning(`sleep.tasks ${request.detector} refused a mint: evidence not found in the source it cites`);
|
|
11507
|
+
return "unverified";
|
|
11508
|
+
}
|
|
11509
|
+
const frame = frameKeyOf(request.claim);
|
|
11510
|
+
if (frame !== null && open.some((detected) => frameKeyOf(detected.claim) === frame)) return "framed";
|
|
11511
|
+
if (budget.remaining <= 0) {
|
|
11512
|
+
budget.overflow += 1;
|
|
11513
|
+
return "capped";
|
|
11514
|
+
}
|
|
11515
|
+
const title = titleOf(request.title);
|
|
11516
|
+
const path = detectedTaskPath(key, title);
|
|
11517
|
+
yield* writeFileBytes(env, path, renderTemplate({
|
|
11518
|
+
title,
|
|
11519
|
+
/**
|
|
11520
|
+
* `claim` is required by `NewMemoryInput` and UNREAD when `articleHtml` is present, since
|
|
11521
|
+
* `articleHtmlFor` returns the pre-authored markup verbatim. It is stated anyway rather than
|
|
11522
|
+
* stubbed, so the two never disagree about what this file's claim is: {@link detectedArticle}
|
|
11523
|
+
* builds the `<mark>` from this same value.
|
|
11524
|
+
*/
|
|
11525
|
+
claim: request.claim,
|
|
11526
|
+
articleHtml: detectedArticle(env, request),
|
|
11527
|
+
memoryType: "task",
|
|
11528
|
+
taskStatus: "todo",
|
|
11529
|
+
at: env.at,
|
|
11530
|
+
/**
|
|
11531
|
+
* `agent:sleep`, which is the author separation issue #44 asks for: a human's queue and the
|
|
11532
|
+
* machine's are told apart by `memhtml-author` rather than by where they sit, so both live in
|
|
11533
|
+
* one list and `task list` can filter.
|
|
11534
|
+
*/
|
|
11535
|
+
author: "agent:sleep",
|
|
11536
|
+
/**
|
|
11537
|
+
* The generic tag first and the detector second, so the pair reads as "detected, by this".
|
|
11538
|
+
* The ORDER is load-bearing: `memhtml-tag` is repeatable, the serializer emits repeatables in
|
|
11539
|
+
* the order given, and {@link openDetections} reads the detector back off the second value.
|
|
11540
|
+
*/
|
|
11541
|
+
tags: [DETECTED_TAG, request.detector],
|
|
11542
|
+
/**
|
|
11543
|
+
* `from_session` provenance, per issue #44, as the ordinary `memhtml-session` meta rather than
|
|
11544
|
+
* anything new. It is already in the closed vocabulary, already projects to
|
|
11545
|
+
* `files.session_id`, and already carries exactly this meaning on a memory an agent wrote
|
|
11546
|
+
* during a session — so a detected task minted from a transcript answers "which session is
|
|
11547
|
+
* this from" through the same column every other provenance query reads. Only the `session`
|
|
11548
|
+
* evidence arm has one; a measurement and a corpus quote are not from a session, and stamping
|
|
11549
|
+
* the run's id there would make the column mean two things.
|
|
11550
|
+
*/
|
|
11551
|
+
...request.evidence.kind === "session" ? { sessionId: request.evidence.sessionId.trim() } : {},
|
|
11552
|
+
...dueOf(request.dueHint) === void 0 ? {} : { dueAt: dueOf(request.dueHint) }
|
|
11553
|
+
}));
|
|
11554
|
+
yield* env.deps.git.add([path]);
|
|
11555
|
+
budget.remaining -= 1;
|
|
11556
|
+
return "minted";
|
|
11557
|
+
});
|
|
11558
|
+
/**
|
|
11559
|
+
* Close every open detection of one detector whose key is not in `liveKeys`: stamp `done` and archive.
|
|
11560
|
+
*
|
|
11561
|
+
* Self-cleaning, per issue #44: "a finding that stops appearing closes its task with reason
|
|
11562
|
+
* `no longer detected`". A queue that only grows is a queue a human abandons, and the finding is the
|
|
11563
|
+
* only thing that can say a review is no longer wanted — the human declining to act on it cannot,
|
|
11564
|
+
* because that is indistinguishable from not having got to it yet.
|
|
11565
|
+
*
|
|
11566
|
+
* **`done` plus archive, matching `memhtml task status done` exactly.** `apps/cli/src/operations.ts`
|
|
11567
|
+
* stamps the status and then routes through `store.archiveMemory`, and this does the same two things
|
|
11568
|
+
* through sleep's staging discipline instead of through the store's own commit: the stamp goes on
|
|
11569
|
+
* first so it travels with the `git mv`, and `archiveFile` re-writes the stamped bytes at the
|
|
11570
|
+
* destination. `done` is not a resting state on its own; the archive tree plus `git log` is what
|
|
11571
|
+
* answers "what did I close".
|
|
11572
|
+
*
|
|
11573
|
+
* **The prose reason lives in the commit; the MACHINE/HUMAN distinction lives in a tag.** There is no
|
|
11574
|
+
* `memhtml-*` name for a closing reason and the vocabulary is closed, so the caller's `commitPhase` body
|
|
11575
|
+
* carries `no longer detected` — which is also where `store.archiveMemory` puts its own reason. But that
|
|
11576
|
+
* text is not readable as a fact about the file, and {@link mintDetectedTask}'s dismissal check needs one
|
|
11577
|
+
* bit of it: was this closed by the system or by a person. So every closure here also appends
|
|
11578
|
+
* {@link MACHINE_CLOSED_TAG}, which makes a swept task re-mintable when its finding comes back while a
|
|
11579
|
+
* human's closure stands. See {@link MACHINE_CLOSED_TAG}.
|
|
11580
|
+
*
|
|
11581
|
+
* **`liveKeys` must be every finding the detector SAW, not every finding it minted.** A finding
|
|
11582
|
+
* turned away by the cap is still live, and closing its task because a busy night declined to
|
|
11583
|
+
* refresh it would delete a real review the moment the queue got full. A finding below a phase's own
|
|
11584
|
+
* CONFIDENCE floor was also seen, so it belongs there too.
|
|
11585
|
+
*
|
|
11586
|
+
* **Call this only on the night's full-strength path.** A phase that degraded — no model bound, a
|
|
11587
|
+
* batch whose call failed, an early return before its scan finished — did not evaluate the candidate
|
|
11588
|
+
* set, so its `liveKeys` describes what it managed to look at rather than what exists. Sweeping there
|
|
11589
|
+
* would close a human's queue every credential-free night. Each caller states the condition it
|
|
11590
|
+
* sweeps under, and a caller whose degraded arm mines at a DIFFERENT floor than its full-strength one
|
|
11591
|
+
* is degraded even when no call failed — `dedup-merge`'s no-model arm records that reading.
|
|
11592
|
+
*/
|
|
11593
|
+
const closeVanishedDetections = (env, detector, liveKeys) => Effect.gen(function* () {
|
|
11594
|
+
const open = yield* openDetections(env);
|
|
11595
|
+
let closed = 0;
|
|
11596
|
+
for (const detected of open) {
|
|
11597
|
+
if (detected.detector !== detector) continue;
|
|
11598
|
+
if (liveKeys.has(detected.key)) continue;
|
|
11599
|
+
yield* stampFile(env, detected.path, [
|
|
11600
|
+
meta("memhtml-task-status", "done"),
|
|
11601
|
+
meta("memhtml-updated", env.at),
|
|
11602
|
+
addTag(MACHINE_CLOSED_TAG)
|
|
11603
|
+
]);
|
|
11604
|
+
if ((yield* archiveFile(env, detected.path)) !== null) closed += 1;
|
|
11605
|
+
}
|
|
11606
|
+
return closed;
|
|
11607
|
+
});
|
|
11608
|
+
/**
|
|
11609
|
+
* Close ONE detected task by path: stamp `done` and archive, exactly as {@link closeVanishedDetections}
|
|
11610
|
+
* does per file. Answers `false` and writes nothing when the path is not a detected task's.
|
|
11611
|
+
*
|
|
11612
|
+
* The refusal is the point, and it is a HARD guard rather than a convention. Surface 2 closes a task
|
|
11613
|
+
* because a transcript says the work is done, which is a model's reading of somebody's prose — so this
|
|
11614
|
+
* is the one closure path whose trigger is not a fact the corpus can check. A human-opened task closed
|
|
11615
|
+
* on that basis is work silently taken out of somebody's queue by a sentence they did not write, and
|
|
11616
|
+
* `done` ARCHIVES, so the file also leaves the directory they look in. {@link isDetectedTaskPath} is
|
|
11617
|
+
* the discriminator because it reads the PATH: it needs no parse, no index row, and no meta, so it
|
|
11618
|
+
* cannot be defeated by a file whose head a model influenced.
|
|
11619
|
+
*
|
|
11620
|
+
* A caller that found its path through {@link openDetections} is already inside the guard, since that
|
|
11621
|
+
* function only returns detected paths. The check runs anyway, here, at the write: a second caller
|
|
11622
|
+
* arriving with a path from a query, a report, or a match on a title is the case this exists for, and a
|
|
11623
|
+
* guard that lived at the lookup instead would not cover it.
|
|
11624
|
+
*
|
|
11625
|
+
* The closing REASON goes in the caller's commit body, for the reason
|
|
11626
|
+
* {@link closeVanishedDetections} records: there is no `memhtml-*` name for it and the vocabulary is
|
|
11627
|
+
* closed. {@link MACHINE_CLOSED_TAG} rides on the file here too, and it is if anything MORE load-bearing
|
|
11628
|
+
* on this path: a completion detected in a transcript is the machine's reading, so if the commitment is
|
|
11629
|
+
* restated on a later night the task must be re-mintable rather than read as a human's dismissal.
|
|
11630
|
+
*/
|
|
11631
|
+
const closeDetectedTask = (env, path) => Effect.gen(function* () {
|
|
11632
|
+
if (!isDetectedTaskPath(path)) {
|
|
11633
|
+
yield* Effect.logWarning(`sleep.tasks refused to close ${path}: not a detected task, so no detector may close it`);
|
|
11634
|
+
return false;
|
|
11635
|
+
}
|
|
11636
|
+
yield* stampFile(env, path, [
|
|
11637
|
+
meta("memhtml-task-status", "done"),
|
|
11638
|
+
meta("memhtml-updated", env.at),
|
|
11639
|
+
addTag(MACHINE_CLOSED_TAG)
|
|
11640
|
+
]);
|
|
11641
|
+
return (yield* archiveFile(env, path)) !== null;
|
|
11642
|
+
});
|
|
11643
|
+
/**
|
|
11644
|
+
* Every OPEN detected task, read from the tree, path-ordered.
|
|
11645
|
+
*
|
|
11646
|
+
* Open means present under {@link DETECTED_TASK_DIR} and not stamped `done`. Both halves are needed
|
|
11647
|
+
* and neither is redundant: closing archives the file out of the directory, so presence is almost
|
|
11648
|
+
* sufficient — but a human may stamp `done` by hand through `memhtml task status`, or a run may be
|
|
11649
|
+
* interrupted between the stamp and the `git mv`, and a detector must not refresh or re-close a task
|
|
11650
|
+
* somebody already finished.
|
|
11651
|
+
*
|
|
11652
|
+
* **Parsed, not scanned for meta lines.** `memhtml-tag` is repeatable and the surgical `readMeta`
|
|
11653
|
+
* returns only the first value of a name, so the detector tag is unreachable without the real parser
|
|
11654
|
+
* — the same reason `entity-resolution`'s alias oracle parses person files. A file that does not
|
|
11655
|
+
* parse is skipped: it is not indexed either, so no detection is keyed on it.
|
|
11656
|
+
*/
|
|
11657
|
+
const openDetections = (env) => Effect.gen(function* () {
|
|
11658
|
+
const filenames = yield* detectedFilenames(env, DETECTED_TASK_DIR);
|
|
11659
|
+
const out = [];
|
|
11660
|
+
for (const filename of filenames) {
|
|
11661
|
+
const path = `${DETECTED_TASK_DIR}/${filename}`;
|
|
11662
|
+
const key = detectionKeyOf(path);
|
|
11663
|
+
if (key === void 0) continue;
|
|
11664
|
+
const html = yield* readFileBytes(env, path);
|
|
11665
|
+
if (html === void 0) continue;
|
|
11666
|
+
const doc = yield* parseMemory(html).pipe(Effect.orElseSucceed(() => void 0));
|
|
11667
|
+
if (doc === void 0) continue;
|
|
11668
|
+
if (doc.metas.memoryType !== "task" || doc.metas.taskStatus === "done") continue;
|
|
11669
|
+
const [first, second] = doc.tags;
|
|
11670
|
+
if (first !== "detected" || second === void 0) continue;
|
|
11671
|
+
out.push({
|
|
11672
|
+
path,
|
|
11673
|
+
key,
|
|
11674
|
+
detector: second,
|
|
11675
|
+
title: doc.title,
|
|
11676
|
+
claim: doc.article.gist
|
|
11677
|
+
});
|
|
11678
|
+
}
|
|
11679
|
+
return out;
|
|
11680
|
+
});
|
|
11681
|
+
/**
|
|
11682
|
+
* True when a HUMAN closed a detected task of this key and the archive still holds it: a standing
|
|
11683
|
+
* dismissal. See the module header for what makes it durable and how a human takes it back.
|
|
11684
|
+
*
|
|
11685
|
+
* A match requires all four, and each rules out a different false positive:
|
|
11686
|
+
*
|
|
11687
|
+
* - the filename carries this exact key, which is a prefix match on the stem rather than one derived
|
|
11688
|
+
* path, because `archiveFile`'s ordinal suffixing lets one key own several archived files;
|
|
11689
|
+
* - the file parses as a `task` stamped `done`, so an archived task somebody left `todo` (a file moved
|
|
11690
|
+
* by hand, or a run interrupted between the stamp and the `git mv`) is not read as an answer;
|
|
11691
|
+
* - the first tag is {@link DETECTED_TAG}, so a hand-written task that happened to land on a `det-` name
|
|
11692
|
+
* cannot dismiss a detector's finding;
|
|
11693
|
+
* - and {@link MACHINE_CLOSED_TAG} is ABSENT, which is the whole discriminator: a swept task carries it
|
|
11694
|
+
* and must stay re-mintable.
|
|
11695
|
+
*
|
|
11696
|
+
* The scan STOPS at the first match, so the common case on a corpus with archives costs one `readdir`
|
|
11697
|
+
* of the current year. Every year is a separate `readdir` and a missing one is empty, so a corpus with no
|
|
11698
|
+
* archive at all pays {@link DISMISSAL_LOOKBACK_YEARS} ENOENTs and reads no file.
|
|
11699
|
+
*/
|
|
11700
|
+
const humanDismissed = (env, key) => Effect.gen(function* () {
|
|
11701
|
+
const year = yearOf(env.date);
|
|
11702
|
+
for (let back = 0; back <= 10; back += 1) {
|
|
11703
|
+
const directory = archivePathFor(DETECTED_TASK_DIR, year - back);
|
|
11704
|
+
for (const filename of yield* detectedFilenames(env, directory)) {
|
|
11705
|
+
if (!filename.startsWith(`${key}-`)) continue;
|
|
11706
|
+
const html = yield* readFileBytes(env, `${directory}/${filename}`);
|
|
11707
|
+
if (html === void 0) continue;
|
|
11708
|
+
const doc = yield* parseMemory(html).pipe(Effect.orElseSucceed(() => void 0));
|
|
11709
|
+
if (doc === void 0) continue;
|
|
11710
|
+
if (doc.metas.memoryType !== "task" || doc.metas.taskStatus !== "done") continue;
|
|
11711
|
+
if (doc.tags[0] !== "detected") continue;
|
|
11712
|
+
if (doc.tags.includes("machine-closed")) continue;
|
|
11713
|
+
return true;
|
|
11714
|
+
}
|
|
11715
|
+
}
|
|
11716
|
+
return false;
|
|
11717
|
+
});
|
|
11718
|
+
/**
|
|
11719
|
+
* The `.html` filenames under one directory carrying the detection prefix, sorted.
|
|
11720
|
+
*
|
|
11721
|
+
* A missing directory is `[]`, not a failure: a corpus that has never had a detected task has no
|
|
11722
|
+
* `areas/inbox/tasks` at all, and that is the state every first night starts from. It is the normal case
|
|
11723
|
+
* for the archive year buckets too, where most of the lookback window will never exist. Any OTHER errno
|
|
11724
|
+
* still fails, because a permission error on the queue directory is a real fault that must not read
|
|
11725
|
+
* as an empty queue and take a sweep through every open detection.
|
|
11726
|
+
*/
|
|
11727
|
+
const detectedFilenames = (env, directory) => attemptIo(`sleep.tasks.list:${directory}`, async () => {
|
|
11728
|
+
const { readdir } = await import("node:fs/promises");
|
|
11729
|
+
try {
|
|
11730
|
+
return (await readdir(absoluteIn(env, directory))).filter((name) => name.startsWith("det-") && name.endsWith(".html")).sort();
|
|
11731
|
+
} catch (cause) {
|
|
11732
|
+
if (cause.code === "ENOENT") return [];
|
|
11733
|
+
throw cause;
|
|
11734
|
+
}
|
|
11735
|
+
});
|
|
11736
|
+
/**
|
|
11737
|
+
* True when the evidence is admissible: a quote only when the cited file's own article text carries it,
|
|
11738
|
+
* a measurement or a session citation whenever it is non-empty.
|
|
11739
|
+
*
|
|
11740
|
+
* Compared with whitespace collapsed on BOTH sides, and case-sensitively. Whitespace is not content
|
|
11741
|
+
* here — the same sentence read out of a `body_text` projection, out of a re-wrapped paragraph, and
|
|
11742
|
+
* out of the file's markup differ only in spacing, and refusing on that would refuse true quotes.
|
|
11743
|
+
* Case IS content: "the deploy is safe" and "the deploy is SAFE" are the same words and a citation
|
|
11744
|
+
* that changed the emphasis is not verbatim.
|
|
11745
|
+
*
|
|
11746
|
+
* The check reads the FILE, not the index row the caller found the sentence in. The tree is the
|
|
11747
|
+
* system of record and the index is refreshed once per night, so a row can name text an earlier
|
|
11748
|
+
* phase's commit has already replaced. A missing file refuses, which is the same posture every
|
|
11749
|
+
* other phase takes toward a path the tree no longer holds.
|
|
11750
|
+
*
|
|
11751
|
+
* A `session` citation has no file to read and this function says so rather than pretending to check
|
|
11752
|
+
* one. What stands behind it is `ungroundedCommitmentReason` at the client boundary plus the phase's
|
|
11753
|
+
* own batch-membership check; the module header records why re-reading the transcript here was
|
|
11754
|
+
* declined. The non-empty test is not the guard, it is the same floor the other two arms carry.
|
|
11755
|
+
*/
|
|
11756
|
+
const evidenceHolds = (env, evidence) => Effect.gen(function* () {
|
|
11757
|
+
if (evidence.kind === "measurement") return evidence.detail.trim() !== "";
|
|
11758
|
+
if (evidence.kind === "session") return evidence.sessionId.trim() !== "" && evidence.statement.trim() !== "";
|
|
11759
|
+
const quote = flatten(evidence.quote);
|
|
11760
|
+
if (quote === "") return false;
|
|
11761
|
+
const html = yield* readFileBytes(env, evidence.sourcePath);
|
|
11762
|
+
if (html === void 0) return false;
|
|
11763
|
+
const doc = yield* parseMemory(html).pipe(Effect.orElseSucceed(() => void 0));
|
|
11764
|
+
if (doc === void 0) return false;
|
|
11765
|
+
return flatten(doc.article.bodyText).includes(quote);
|
|
11766
|
+
});
|
|
11767
|
+
/** Whitespace collapsed to single spaces and trimmed. The comparison form for a quote. */
|
|
11768
|
+
const flatten = (text) => text.replace(/\s+/g, " ").trim();
|
|
11769
|
+
/** A title: one line, sentence punctuation kept, cut to {@link TITLE_CHARS}. */
|
|
11770
|
+
const titleOf = (title) => flatten(title).slice(0, TITLE_CHARS).trim();
|
|
11771
|
+
/**
|
|
11772
|
+
* A due hint the format accepts, or `undefined`.
|
|
11773
|
+
*
|
|
11774
|
+
* `memhtml-due` is compared and ordered AS A STRING by the overdue query, so a value that does not
|
|
11775
|
+
* sort alongside the others would corrupt it. `isValidDatetime` is the format's own predicate, so a
|
|
11776
|
+
* hint a model supplied is dropped rather than written and the task simply has no due date.
|
|
11777
|
+
*/
|
|
11778
|
+
const dueOf = (hint) => hint !== void 0 && isValidDatetime(hint.trim()) ? hint.trim() : void 0;
|
|
11779
|
+
/**
|
|
11780
|
+
* A detected task's article: the claim, the detail, the evidence, and the provenance line.
|
|
11781
|
+
*
|
|
11782
|
+
* Authored as MARKUP rather than through the template's prose path, because the evidence needs
|
|
11783
|
+
* `<q cite>` — the vocabulary's own quotation element, which carries its source URI and projects into
|
|
11784
|
+
* `file_citations(text, href)`. That is what makes issue #44's "the parser can verify the quote still
|
|
11785
|
+
* exists in the cited source" a single query rather than a re-read of every task. `<blockquote>` is
|
|
11786
|
+
* NOT in the closed vocabulary, so the quote is inline in its own paragraph.
|
|
11787
|
+
*
|
|
11788
|
+
* Using `articleHtml` means this function owns constraint 1, so the `<mark>` is placed in the first
|
|
11789
|
+
* `<p>` here and nowhere else. Every interpolation goes through `escapeText`/`escapeAttribute`: a
|
|
11790
|
+
* model-supplied sentence reaches this string on surface 3, and the source path reaches it on all of
|
|
11791
|
+
* them.
|
|
11792
|
+
*/
|
|
11793
|
+
const detectedArticle = (env, request) => {
|
|
11794
|
+
const paragraphs = [`<p><mark>${escapeText(flatten(request.claim))}</mark></p>`];
|
|
11795
|
+
if (request.detail !== void 0 && request.detail.trim() !== "") paragraphs.push(`<p>${escapeText(flatten(request.detail))}</p>`);
|
|
11796
|
+
paragraphs.push(evidenceParagraph(request.evidence));
|
|
11797
|
+
paragraphs.push(`<p>Detected by <code>${escapeText(request.detector)}</code> on run <code>${escapeText(env.runId)}</code>. This is a proposal for a human to decide, not a finding the corpus asserts. It closes itself when the detector stops seeing it.</p>`);
|
|
11798
|
+
return paragraphs.join("\n");
|
|
11799
|
+
};
|
|
11800
|
+
/**
|
|
11801
|
+
* The one paragraph that states what the finding rests on, per evidence kind.
|
|
11802
|
+
*
|
|
11803
|
+
* Split out of {@link detectedArticle} once the third arm arrived, so the three readings sit beside
|
|
11804
|
+
* each other and the difference between them is legible. Each says out loud what a reader can do with
|
|
11805
|
+
* it: open the file and find the sentence, take the number on the corpus's word, or go back to the
|
|
11806
|
+
* session and read the line in the commit that opened this.
|
|
11807
|
+
*
|
|
11808
|
+
* The `session` arm carries NO quote, which is the trace-plane invariant and not an omission. See the
|
|
11809
|
+
* module header. It also carries no `<q cite>`, because there is nothing to cite: a session is not a
|
|
11810
|
+
* corpus path and `hrefFor` over an id would produce a link that resolves nowhere, which
|
|
11811
|
+
* `integrity`'s dangling-edge repair exists to prevent.
|
|
11812
|
+
*/
|
|
11813
|
+
const evidenceParagraph = (evidence) => {
|
|
11814
|
+
if (evidence.kind === "quote") return `<p>Evidence, verbatim from <code>${escapeText(evidence.sourcePath)}</code>: <q cite="${escapeAttribute(hrefFor(evidence.sourcePath))}">${escapeText(flatten(evidence.quote))}</q></p>`;
|
|
11815
|
+
if (evidence.kind === "session") return `<p>Evidence, from session <code>${escapeText(evidence.sessionId)}</code>: ${escapeText(flatten(evidence.statement))} The verbatim line is in the commit that opened this task; a transcript span is not stored in the corpus.</p>`;
|
|
11816
|
+
return `<p>Evidence, measured over the corpus: ${escapeText(flatten(evidence.detail))}</p>`;
|
|
11817
|
+
};
|
|
11818
|
+
|
|
10806
11819
|
//#endregion
|
|
10807
11820
|
//#region packages/sleep/dist/phases/dedup-merge.js
|
|
10808
11821
|
/**
|
|
@@ -10861,6 +11874,16 @@ const confidenceDecay = (env) => Effect.gen(function* () {
|
|
|
10861
11874
|
* One commit for the whole batch, not one per pair. A keeper's `memhtml-supersedes` points at its
|
|
10862
11875
|
* dropped file's ARCHIVE path, which is where that file lives only after this commit lands.
|
|
10863
11876
|
* Splitting them would create a dangling href in the commit that made it dangle.
|
|
11877
|
+
*
|
|
11878
|
+
* ## A vetoed pair becomes a review task
|
|
11879
|
+
*
|
|
11880
|
+
* Surface 1 of issue #44, second detector. The veto is the phase's strongest signal that something
|
|
11881
|
+
* needs a HUMAN rather than a merge: two memories a cosine says are the same and a divergence
|
|
11882
|
+
* predicate says cannot both be true is either a correction the corpus has not recorded as one, or a
|
|
11883
|
+
* pair of facts about different things that read alike. Neither resolution is a nightly job's to make —
|
|
11884
|
+
* choosing the winner of a contradiction is a one-way door on stored belief — and the count alone told
|
|
11885
|
+
* nobody. {@link mintVetoTasks} opens one task per vetoed pair NAMING THE PREDICATE that fired, in the
|
|
11886
|
+
* same commit as the folds.
|
|
10864
11887
|
*/
|
|
10865
11888
|
/**
|
|
10866
11889
|
* The mining floor when a model is bound. RECALL-oriented, and deliberately below the merge floor.
|
|
@@ -11002,12 +12025,41 @@ const dedupMerge = (env) => Effect.gen(function* () {
|
|
|
11002
12025
|
* could call a model, which is what makes the existing dedup tests an oracle for the rest.
|
|
11003
12026
|
*/
|
|
11004
12027
|
const decisions = mergeCandidates(oriented);
|
|
11005
|
-
return yield* commitMerges(
|
|
11006
|
-
|
|
11007
|
-
|
|
11008
|
-
|
|
11009
|
-
|
|
11010
|
-
|
|
12028
|
+
return yield* commitMerges(
|
|
12029
|
+
env,
|
|
12030
|
+
decisions,
|
|
12031
|
+
{
|
|
12032
|
+
candidates: oriented.length,
|
|
12033
|
+
components: 0,
|
|
12034
|
+
llmGroups: 0,
|
|
12035
|
+
vetoed: oriented.length - decisions.length
|
|
12036
|
+
},
|
|
12037
|
+
/**
|
|
12038
|
+
* Every mined pair on this arm cleared 0.92, so a vetoed one here is a near-certain duplicate
|
|
12039
|
+
* the divergence predicates refused — which is exactly the pair issue #44 wants a human to
|
|
12040
|
+
* look at.
|
|
12041
|
+
*
|
|
12042
|
+
* **`judged: false`, so a modelless night MINTS but never SWEEPS.** No model call failed here,
|
|
12043
|
+
* so an earlier reading called this arm full-strength — and that reading closed human queues.
|
|
12044
|
+
* The two arms mine at DIFFERENT floors: this one at `NEAR_DUPLICATE_THRESHOLD` (0.92) and the
|
|
12045
|
+
* model arm at `DEDUP_COMPONENT_FLOOR` (0.86). A pair vetoed between 0.86 and 0.92 on a night
|
|
12046
|
+
* with credentials is INVISIBLE to this arm — not gone, just below the floor it can see — so its
|
|
12047
|
+
* `liveKeys` omits that pair's key and the sweep would archive a real review because the run
|
|
12048
|
+
* happened to have no credentials. That is exactly what `tasks.ts`'s
|
|
12049
|
+
* `closeVanishedDetections` contract forbids: "sweeping there would close a human's queue every
|
|
12050
|
+
* credential-free night."
|
|
12051
|
+
*
|
|
12052
|
+
* The asymmetry is stated rather than repaired, because it cannot be repaired here: mining this
|
|
12053
|
+
* arm at 0.86 would widen what a no-model night MERGES, and the deterministic floor is the one
|
|
12054
|
+
* number this phase's safety rests on. So the degraded night does the half it can do honestly —
|
|
12055
|
+
* open a task for every divergence it can see — and leaves closure to a night that evaluated the
|
|
12056
|
+
* whole candidate set.
|
|
12057
|
+
*/
|
|
12058
|
+
{
|
|
12059
|
+
vetoed: vetoedPairs(oriented),
|
|
12060
|
+
judged: false
|
|
12061
|
+
}
|
|
12062
|
+
);
|
|
11011
12063
|
}
|
|
11012
12064
|
/**
|
|
11013
12065
|
* The component graph: the mined edges at the recall floor, unioned with the frame-key exact
|
|
@@ -11131,16 +12183,159 @@ const dedupMerge = (env) => Effect.gen(function* () {
|
|
|
11131
12183
|
const proposed = [...groupPairs, ...remaining];
|
|
11132
12184
|
const decisions = mergeCandidates(proposed, { threshold: 0 });
|
|
11133
12185
|
return {
|
|
11134
|
-
...yield* commitMerges(
|
|
11135
|
-
|
|
11136
|
-
|
|
11137
|
-
|
|
11138
|
-
|
|
11139
|
-
|
|
11140
|
-
|
|
12186
|
+
...yield* commitMerges(
|
|
12187
|
+
env,
|
|
12188
|
+
decisions,
|
|
12189
|
+
{
|
|
12190
|
+
candidates: proposed.length,
|
|
12191
|
+
components: components.length,
|
|
12192
|
+
llmGroups,
|
|
12193
|
+
vetoed: proposed.length - decisions.length,
|
|
12194
|
+
skipped
|
|
12195
|
+
},
|
|
12196
|
+
/**
|
|
12197
|
+
* On this arm a vetoed pair is one the MODEL grouped as the same memory, or one that cleared
|
|
12198
|
+
* 0.92 with no group claiming it, and the veto then refused it for a divergence. Both readings
|
|
12199
|
+
* are the issue's case: a semantic reader said "same" and a deterministic predicate said "these
|
|
12200
|
+
* differ in a way that matters", and the resolution — is one a correction of the other? — is a
|
|
12201
|
+
* human's.
|
|
12202
|
+
*
|
|
12203
|
+
* `judged` is false when a batch's call failed, because those components were never partitioned:
|
|
12204
|
+
* their pairs reach the veto only through the mined arm, so a night that lost a call cannot say
|
|
12205
|
+
* whether a pair it did not see is still a candidate.
|
|
12206
|
+
*/
|
|
12207
|
+
{
|
|
12208
|
+
vetoed: vetoedPairs(proposed),
|
|
12209
|
+
judged: skipped === 0
|
|
12210
|
+
}
|
|
12211
|
+
),
|
|
11141
12212
|
llmCalls
|
|
11142
12213
|
};
|
|
11143
12214
|
});
|
|
12215
|
+
/** The detector name every near-duplicate review task is keyed and swept under. */
|
|
12216
|
+
const DEDUP_REVIEW_DETECTOR = "dedup-merge";
|
|
12217
|
+
/**
|
|
12218
|
+
* The proposed pairs the divergence veto refused, with WHICH predicate fired.
|
|
12219
|
+
*
|
|
12220
|
+
* The three predicates are pure, exported, and independently callable, so the phase can name the one
|
|
12221
|
+
* that fired instead of reporting "vetoed". That distinction is the whole value of the task: "these two
|
|
12222
|
+
* carry different numbers" tells a reviewer to compare the numbers, "exactly one of them is negated"
|
|
12223
|
+
* tells them one is probably a correction of the other, and "vetoed" tells them to read both files from
|
|
12224
|
+
* scratch.
|
|
12225
|
+
*
|
|
12226
|
+
* Re-running the predicates rather than threading a reason out of `mergeCandidates` keeps the domain
|
|
12227
|
+
* filter's signature alone: it returns the decisions it made, and asking it to also return a
|
|
12228
|
+
* per-refusal reason would make every caller carry a channel one caller reads. The predicates are pure
|
|
12229
|
+
* token-set comparisons over text already in memory, and this runs over the proposed set once.
|
|
12230
|
+
*
|
|
12231
|
+
* A pair with either text missing is NOT vetoed — the filter skips the veto for it too, since it cannot
|
|
12232
|
+
* evaluate one — so those are absent here, which is correct: an unevaluated pair is not a divergence
|
|
12233
|
+
* anyone found.
|
|
12234
|
+
*/
|
|
12235
|
+
const vetoedPairs = (proposed) => proposed.flatMap((pair) => {
|
|
12236
|
+
const keepText = pair.keepText;
|
|
12237
|
+
const dropText = pair.dropText;
|
|
12238
|
+
if (keepText === void 0 || dropText === void 0) return [];
|
|
12239
|
+
const predicates = [
|
|
12240
|
+
...negationDivergent(keepText, dropText) ? ["one side is negated and the other is not"] : [],
|
|
12241
|
+
...numericTokenDivergent(keepText, dropText) ? ["the two carry different numbers"] : [],
|
|
12242
|
+
...variantQualifierDivergent(keepText, dropText) ? ["the two name different product variants"] : []
|
|
12243
|
+
];
|
|
12244
|
+
if (predicates.length === 0) return [];
|
|
12245
|
+
return [{
|
|
12246
|
+
keepPath: pair.keepPath,
|
|
12247
|
+
dropPath: pair.dropPath,
|
|
12248
|
+
similarity: pair.similarity,
|
|
12249
|
+
predicates
|
|
12250
|
+
}];
|
|
12251
|
+
});
|
|
12252
|
+
/**
|
|
12253
|
+
* Mint one review task per vetoed pair, and sweep the ones that stopped diverging.
|
|
12254
|
+
*
|
|
12255
|
+
* **The key is the two PATHS sorted.** A path is the id of a memory in this corpus, and the question is
|
|
12256
|
+
* about these two files — so unlike the merge itself, which orients keeper-then-drop from corpus dates,
|
|
12257
|
+
* the review question is unordered and sorting is what makes tomorrow's `(b, a)` key with today's
|
|
12258
|
+
* `(a, b)`.
|
|
12259
|
+
*
|
|
12260
|
+
* **The evidence is a MEASUREMENT.** The predicate that fired is a fact about the two token sets, and
|
|
12261
|
+
* no sentence in either file states it. The paths ride in the detail so a reviewer can open both.
|
|
12262
|
+
*
|
|
12263
|
+
* A pair whose veto STOPS firing — because a human corrected one of the two, or because one was
|
|
12264
|
+
* archived — is closed by the sweep, which is right: the divergence was the finding, and it is gone.
|
|
12265
|
+
*
|
|
12266
|
+
* **The claim NAMES THE TWO PATHS, and that is a correctness requirement rather than a nicety.** See
|
|
12267
|
+
* {@link vetoClaim}.
|
|
12268
|
+
*/
|
|
12269
|
+
const mintVetoTasks = (env, vetoed, judged) => Effect.gen(function* () {
|
|
12270
|
+
const budget = budgetFor(env);
|
|
12271
|
+
/**
|
|
12272
|
+
* Keyed and de-duplicated before minting, then walked in key order, so which pairs a budget-capped
|
|
12273
|
+
* night surfaces is a function of the pairs rather than of the arm that proposed them.
|
|
12274
|
+
*/
|
|
12275
|
+
const byKey = /* @__PURE__ */ new Map();
|
|
12276
|
+
for (const pair of vetoed) {
|
|
12277
|
+
const key = detectionKey(DEDUP_REVIEW_DETECTOR, vetoFinding(pair));
|
|
12278
|
+
if (!byKey.has(key)) byKey.set(key, pair);
|
|
12279
|
+
}
|
|
12280
|
+
let minted = 0;
|
|
12281
|
+
let refreshed = 0;
|
|
12282
|
+
let framed = 0;
|
|
12283
|
+
let dismissed = 0;
|
|
12284
|
+
for (const key of [...byKey.keys()].sort()) {
|
|
12285
|
+
const pair = byKey.get(key);
|
|
12286
|
+
if (pair === void 0) continue;
|
|
12287
|
+
const outcome = yield* mintDetectedTask(env, budget, {
|
|
12288
|
+
detector: DEDUP_REVIEW_DETECTOR,
|
|
12289
|
+
finding: vetoFinding(pair),
|
|
12290
|
+
title: `Review near-duplicates vetoed for divergence: ${basenameOf$1(pair.keepPath)} and ${basenameOf$1(pair.dropPath)}`,
|
|
12291
|
+
claim: vetoClaim(pair),
|
|
12292
|
+
detail: "Sleep refused to fold them because folding keeps the OLDER file, so a blind merge of a correction into the memory it corrects would restore the error the correction was written to fix.",
|
|
12293
|
+
evidence: {
|
|
12294
|
+
kind: "measurement",
|
|
12295
|
+
detail: vetoEvidence(pair)
|
|
12296
|
+
}
|
|
12297
|
+
});
|
|
12298
|
+
if (outcome === "minted") minted += 1;
|
|
12299
|
+
else if (outcome === "refreshed") refreshed += 1;
|
|
12300
|
+
else if (outcome === "framed") framed += 1;
|
|
12301
|
+
else if (outcome === "dismissed") dismissed += 1;
|
|
12302
|
+
}
|
|
12303
|
+
const closed = judged ? yield* closeVanishedDetections(env, DEDUP_REVIEW_DETECTOR, new Set(byKey.keys())) : 0;
|
|
12304
|
+
return {
|
|
12305
|
+
minted,
|
|
12306
|
+
refreshed,
|
|
12307
|
+
framed,
|
|
12308
|
+
dismissed,
|
|
12309
|
+
closed
|
|
12310
|
+
};
|
|
12311
|
+
});
|
|
12312
|
+
/**
|
|
12313
|
+
* The claim a vetoed pair becomes: the work stated as work, with THE TWO PATHS IN IT.
|
|
12314
|
+
*
|
|
12315
|
+
* The paths are in the claim because `mintDetectedTask`'s frame-key proximity check reads it, and a
|
|
12316
|
+
* CONSTANT claim caps this detector's queue at one task. The previous wording — "review:
|
|
12317
|
+
* near-duplicates vetoed for divergence — is one a correction of the other?" — keys (measured against
|
|
12318
|
+
* `frameKeyOf`) on `review: near-duplicates vetoed for divergence — is one a correction of`, a non-null
|
|
12319
|
+
* key EVERY vetoed pair shares. So the first pair minted, the second answered `framed`, and a night that
|
|
12320
|
+
* vetoed nine pairs surfaced one. The check exists to catch two DETECTORS describing one work item in
|
|
12321
|
+
* different words; it must never fire between two findings of one detector, which the digest already
|
|
12322
|
+
* separates.
|
|
12323
|
+
*
|
|
12324
|
+
* With the paths in the value position each pair's frame is its own (measured: three pairs, three
|
|
12325
|
+
* distinct keys), so the check still fires against a differently-worded task about the SAME two files
|
|
12326
|
+
* and no longer fires between different pairs. That is the same shape `entity-resolution` and
|
|
12327
|
+
* `edge-typing` already have — both measured `null`, because their claims put the identities in a
|
|
12328
|
+
* position the rule reads as the frame rather than the value.
|
|
12329
|
+
*
|
|
12330
|
+
* The paths left the `detail` when they arrived here, so a reviewer reads them once.
|
|
12331
|
+
*/
|
|
12332
|
+
const vetoClaim = (pair) => `review: ${pair.keepPath} and ${pair.dropPath} are near-duplicates the divergence veto refused to fold.`;
|
|
12333
|
+
/** The canonical finding string: the two paths, sorted. See {@link mintVetoTasks}. */
|
|
12334
|
+
const vetoFinding = (pair) => pair.keepPath < pair.dropPath ? `${pair.keepPath} ${pair.dropPath}` : `${pair.dropPath} ${pair.keepPath}`;
|
|
12335
|
+
/** The evidence line: which predicates fired, and how near the two bodies measured. */
|
|
12336
|
+
const vetoEvidence = (pair) => `${pair.predicates.join("; ")} — at cosine ${pair.similarity.toFixed(3)}, at or above the ${String(DEDUP_COMPONENT_FLOOR)} candidate floor`;
|
|
12337
|
+
/** A path's filename without its extension, for a title that fits `ls` and a commit subject. */
|
|
12338
|
+
const basenameOf$1 = (path) => path.slice(path.lastIndexOf("/") + 1).replace(/\.html$/, "");
|
|
11144
12339
|
/**
|
|
11145
12340
|
* Archive each drop, stamp each keeper, and commit once.
|
|
11146
12341
|
*
|
|
@@ -11160,16 +12355,20 @@ const dedupMerge = (env) => Effect.gen(function* () {
|
|
|
11160
12355
|
* night two, and a counter bumped by a run that wrote nothing is a night of evidence the corpus never
|
|
11161
12356
|
* saw.
|
|
11162
12357
|
*/
|
|
11163
|
-
const commitMerges = (env, decisions, base) => Effect.gen(function* () {
|
|
11164
|
-
|
|
11165
|
-
|
|
11166
|
-
|
|
11167
|
-
|
|
11168
|
-
|
|
12358
|
+
const commitMerges = (env, decisions, base, review) => Effect.gen(function* () {
|
|
12359
|
+
/**
|
|
12360
|
+
* A dry run counts the folds and the vetoes and mints nothing. Every count above is already real on
|
|
12361
|
+
* a dry run because an operator sizing a night needs them; a TASK is a write, so it waits for a
|
|
12362
|
+
* real night the same way the archives do.
|
|
12363
|
+
*/
|
|
11169
12364
|
if (env.dryRun) return emptyOutcome({
|
|
11170
12365
|
...base,
|
|
11171
12366
|
merged: decisions.length,
|
|
11172
|
-
vanished: 0
|
|
12367
|
+
vanished: 0,
|
|
12368
|
+
tasksMinted: 0,
|
|
12369
|
+
tasksFramed: 0,
|
|
12370
|
+
tasksDismissed: 0,
|
|
12371
|
+
tasksClosed: 0
|
|
11173
12372
|
});
|
|
11174
12373
|
let merged = 0;
|
|
11175
12374
|
let vanished = 0;
|
|
@@ -11182,14 +12381,26 @@ const commitMerges = (env, decisions, base) => Effect.gen(function* () {
|
|
|
11182
12381
|
yield* stampFile(env, decision.keepPath, [link("supersedes", hrefFor(archived)), meta("memhtml-updated", env.at)]);
|
|
11183
12382
|
merged += 1;
|
|
11184
12383
|
}
|
|
12384
|
+
/**
|
|
12385
|
+
* The vetoed pairs become tasks in the SAME commit as the folds, and the mint runs even when
|
|
12386
|
+
* nothing folded — which is why the old `decisions.length === 0` early return is gone. A night
|
|
12387
|
+
* whose every candidate was vetoed is precisely the night with the most for a human to decide, and
|
|
12388
|
+
* returning early on it would have made surface 1 unreachable on exactly that night.
|
|
12389
|
+
*/
|
|
12390
|
+
const tasks = yield* mintVetoTasks(env, review.vetoed, review.judged);
|
|
11185
12391
|
const final = {
|
|
11186
12392
|
...base,
|
|
11187
12393
|
merged,
|
|
11188
|
-
vanished
|
|
12394
|
+
vanished,
|
|
12395
|
+
tasksMinted: tasks.minted,
|
|
12396
|
+
tasksFramed: tasks.framed,
|
|
12397
|
+
tasksDismissed: tasks.dismissed,
|
|
12398
|
+
tasksClosed: tasks.closed
|
|
11189
12399
|
};
|
|
12400
|
+
if (merged === 0 && tasks.minted === 0 && tasks.refreshed === 0 && tasks.closed === 0) return emptyOutcome(final);
|
|
11190
12401
|
return {
|
|
11191
12402
|
counts: final,
|
|
11192
|
-
commitSha: yield* commitPhase(env, "dedup-merge", `fold ${merged} near-duplicates into canonicals`, final),
|
|
12403
|
+
commitSha: yield* commitPhase(env, "dedup-merge", `fold ${merged} near-duplicates into canonicals`, final, tasks.minted + tasks.closed === 0 ? void 0 : `deferred ${tasks.minted} vetoed pairs to review tasks` + (tasks.closed === 0 ? "" : `; closed ${tasks.closed}: no longer detected`)),
|
|
11193
12404
|
llmCalls: 0
|
|
11194
12405
|
};
|
|
11195
12406
|
});
|
|
@@ -11249,6 +12460,13 @@ const commitMerges = (env, decisions, base) => Effect.gen(function* () {
|
|
|
11249
12460
|
* contradiction is a one-way door on stored belief, and it belongs to an agent or a human, not to a
|
|
11250
12461
|
* nightly job.
|
|
11251
12462
|
*
|
|
12463
|
+
* **A single-detection contradiction becomes a TASK.** Surface 1 of issue #44, third detector, and the
|
|
12464
|
+
* one whose gap was widest: a contradiction at `detections = 1` is written nowhere at all, so a real
|
|
12465
|
+
* conflict is invisible for a night and one the model never repeats is invisible forever. The task
|
|
12466
|
+
* names both files and carries the model's confidence and rationale, and it CLOSES on the second night
|
|
12467
|
+
* when the edge is promoted — the corpus then records the conflict where a reader will find it, which
|
|
12468
|
+
* is a better home for it than a to-do item.
|
|
12469
|
+
*
|
|
11252
12470
|
* This phase replaced `conflict-detection`, which asked one `generateObject` per pair for a stance
|
|
11253
12471
|
* verdict over `{contradicts, entails, neutral}`. Contradiction is now one more verdict in the same
|
|
11254
12472
|
* list, with the same corroboration gate. A run whose commits predate the rename carries
|
|
@@ -11414,7 +12632,11 @@ const edgeTyping = (env) => Effect.gen(function* () {
|
|
|
11414
12632
|
promoted: 0,
|
|
11415
12633
|
skipped: 0,
|
|
11416
12634
|
capped: 0,
|
|
11417
|
-
duplicates: 0
|
|
12635
|
+
duplicates: 0,
|
|
12636
|
+
tasksMinted: 0,
|
|
12637
|
+
tasksFramed: 0,
|
|
12638
|
+
tasksDismissed: 0,
|
|
12639
|
+
tasksClosed: 0
|
|
11418
12640
|
};
|
|
11419
12641
|
if (candidates.length === 0) return emptyOutcome(zero);
|
|
11420
12642
|
if (env.dryRun) return emptyOutcome({
|
|
@@ -11476,6 +12698,14 @@ const edgeTyping = (env) => Effect.gen(function* () {
|
|
|
11476
12698
|
/** Second-and-later verdicts naming a key their batch had already answered for. */
|
|
11477
12699
|
let duplicates = 0;
|
|
11478
12700
|
let llmCalls = 0;
|
|
12701
|
+
/**
|
|
12702
|
+
* Contradictions this night detected for the FIRST time, so below the promotion gate.
|
|
12703
|
+
*
|
|
12704
|
+
* A value rather than a count, for the same reason `entity-resolution` keeps its review pairs: a
|
|
12705
|
+
* conflict the gate declined to write is a conflict nobody is told about, and a human deciding
|
|
12706
|
+
* which of two claims survives is exactly what the gate is holding the decision open for.
|
|
12707
|
+
*/
|
|
12708
|
+
const deferred = [];
|
|
11479
12709
|
for (const batch of batches) {
|
|
11480
12710
|
/** Opaque keys again, so a verdict cannot name a path. Each SIDE is sliced to its budget. */
|
|
11481
12711
|
const keyed = keyMembers(batch, (candidate) => pairText(candidate.srcText.slice(0, EDGE_PAIR_SIDE_CHARS), candidate.dstText.slice(0, EDGE_PAIR_SIDE_CHARS)));
|
|
@@ -11529,6 +12759,22 @@ const edgeTyping = (env) => Effect.gen(function* () {
|
|
|
11529
12759
|
dstPath: candidate.pair.dst,
|
|
11530
12760
|
at: env.at
|
|
11531
12761
|
}))[0];
|
|
12762
|
+
/**
|
|
12763
|
+
* A first detection is the case issue #44 asks for a task about: the model asserts these two
|
|
12764
|
+
* claims cannot both be true, and the corroboration gate correctly refuses to write it into
|
|
12765
|
+
* the files on one night's evidence — so a real contradiction sits invisible for a night,
|
|
12766
|
+
* and one the model will not see again sits invisible forever. Deferring it to a human is
|
|
12767
|
+
* the third detector on surface 1, and it is the ONLY one of the three where a second night
|
|
12768
|
+
* resolves the finding on its own: at `detections >= 2` the edge is promoted and the task
|
|
12769
|
+
* closes, because the corpus now records the conflict where a reader will find it.
|
|
12770
|
+
*/
|
|
12771
|
+
if (row !== void 0 && row.detections < 2) deferred.push({
|
|
12772
|
+
src: candidate.pair.src,
|
|
12773
|
+
dst: candidate.pair.dst,
|
|
12774
|
+
confidence: verdict.confidence,
|
|
12775
|
+
detections: row.detections,
|
|
12776
|
+
...verdict.rationale === void 0 ? {} : { rationale: verdict.rationale }
|
|
12777
|
+
});
|
|
11532
12778
|
if (row === void 0 || row.detections < 2 || row.promoted === 1) continue;
|
|
11533
12779
|
if (promoted + typed >= 50) {
|
|
11534
12780
|
capped += 1;
|
|
@@ -11606,6 +12852,13 @@ const edgeTyping = (env) => Effect.gen(function* () {
|
|
|
11606
12852
|
if (yield* stampFile(env, subject, [link(verdict.rel, hrefFor(object)), meta("memhtml-updated", env.at)])) typed += 1;
|
|
11607
12853
|
}
|
|
11608
12854
|
}
|
|
12855
|
+
/**
|
|
12856
|
+
* The single-detection contradictions become tasks in the SAME commit as the promotions. The
|
|
12857
|
+
* sweep is gated on a night that judged its whole candidate set: `skipped` counts pairs whose
|
|
12858
|
+
* batch's call failed as well as pairs whose endpoint the tree no longer holds, and a pair the
|
|
12859
|
+
* model was never asked about must not read as a pair the model stopped contradicting.
|
|
12860
|
+
*/
|
|
12861
|
+
const tasks = yield* mintContradictionTasks(env, deferred, skipped === 0);
|
|
11609
12862
|
const counts = {
|
|
11610
12863
|
candidates: candidates.length,
|
|
11611
12864
|
judged,
|
|
@@ -11614,19 +12867,91 @@ const edgeTyping = (env) => Effect.gen(function* () {
|
|
|
11614
12867
|
promoted,
|
|
11615
12868
|
skipped,
|
|
11616
12869
|
capped,
|
|
11617
|
-
duplicates
|
|
12870
|
+
duplicates,
|
|
12871
|
+
tasksMinted: tasks.minted,
|
|
12872
|
+
tasksFramed: tasks.framed,
|
|
12873
|
+
tasksDismissed: tasks.dismissed,
|
|
12874
|
+
tasksClosed: tasks.closed
|
|
11618
12875
|
};
|
|
11619
|
-
if (promoted === 0 && typed === 0) return {
|
|
12876
|
+
if (promoted === 0 && typed === 0 && tasks.minted === 0 && tasks.refreshed === 0 && tasks.closed === 0) return {
|
|
11620
12877
|
counts,
|
|
11621
12878
|
commitSha: null,
|
|
11622
12879
|
llmCalls
|
|
11623
12880
|
};
|
|
11624
12881
|
return {
|
|
11625
12882
|
counts,
|
|
11626
|
-
commitSha: yield* commitPhase(env, "edge-typing", `promote ${typed} typed edges and ${promoted} corroborated contradictions`, counts),
|
|
12883
|
+
commitSha: yield* commitPhase(env, "edge-typing", `promote ${typed} typed edges and ${promoted} corroborated contradictions`, counts, tasks.minted + tasks.closed === 0 ? void 0 : `deferred ${tasks.minted} single-detection contradictions to review tasks` + (tasks.closed === 0 ? "" : `; closed ${tasks.closed}: no longer detected`)),
|
|
11627
12884
|
llmCalls
|
|
11628
12885
|
};
|
|
11629
12886
|
});
|
|
12887
|
+
/** The detector name every contradiction review task is keyed and swept under. */
|
|
12888
|
+
const EDGE_REVIEW_DETECTOR = "edge-typing";
|
|
12889
|
+
/**
|
|
12890
|
+
* Mint one review task per held-back contradiction, and sweep the ones that stopped being held back.
|
|
12891
|
+
*
|
|
12892
|
+
* **The key is the REL plus the two paths sorted.** The rel is in it because `contradicts` is one of
|
|
12893
|
+
* several verdicts a pair could earn and each would be a different question; the paths are sorted
|
|
12894
|
+
* because a contradiction is symmetric — that symmetry is the phase's own stated reason for promoting
|
|
12895
|
+
* it into both files — so the question is unordered and tonight's `(b, a)` must key with last night's
|
|
12896
|
+
* `(a, b)`.
|
|
12897
|
+
*
|
|
12898
|
+
* **The evidence is a MEASUREMENT even though the model supplied a rationale.** The rationale is prose
|
|
12899
|
+
* ABOUT the two claims, not a span copied out of either, so it would fail the verbatim check on every
|
|
12900
|
+
* mint — and rightly, since the check exists to stop a model's sentence from being presented as a
|
|
12901
|
+
* citation. It rides in the measurement's own text, attributed to the model, where a reader can weigh
|
|
12902
|
+
* it as an opinion.
|
|
12903
|
+
*
|
|
12904
|
+
* **The sweep here is the one whose closure is a good outcome.** A pair promoted on its second night
|
|
12905
|
+
* leaves `deferred`, so the task closes — and the corpus now carries the `contradicts` edge in both
|
|
12906
|
+
* files, which is a better place for the conflict to live than a to-do item. The other two detectors
|
|
12907
|
+
* close when a finding evaporates; this one also closes when the system resolves it.
|
|
12908
|
+
*/
|
|
12909
|
+
const mintContradictionTasks = (env, deferred, judged) => Effect.gen(function* () {
|
|
12910
|
+
const budget = budgetFor(env);
|
|
12911
|
+
const byKey = /* @__PURE__ */ new Map();
|
|
12912
|
+
for (const pending of deferred) {
|
|
12913
|
+
const key = detectionKey(EDGE_REVIEW_DETECTOR, contradictionFinding(pending));
|
|
12914
|
+
if (!byKey.has(key)) byKey.set(key, pending);
|
|
12915
|
+
}
|
|
12916
|
+
let minted = 0;
|
|
12917
|
+
let refreshed = 0;
|
|
12918
|
+
let framed = 0;
|
|
12919
|
+
let dismissed = 0;
|
|
12920
|
+
/** Key order, so which pairs a budget-capped night surfaces is a function of the pairs. */
|
|
12921
|
+
for (const key of [...byKey.keys()].sort()) {
|
|
12922
|
+
const pending = byKey.get(key);
|
|
12923
|
+
if (pending === void 0) continue;
|
|
12924
|
+
const outcome = yield* mintDetectedTask(env, budget, {
|
|
12925
|
+
detector: EDGE_REVIEW_DETECTOR,
|
|
12926
|
+
finding: contradictionFinding(pending),
|
|
12927
|
+
title: `Decide a contradiction between ${basenameOf(pending.src)} and ${basenameOf(pending.dst)}`,
|
|
12928
|
+
claim: `decide: ${pending.src} and ${pending.dst} make claims that cannot both be true.`,
|
|
12929
|
+
detail: `Detected once. The edge is written into both files only at ${String(2)} detections, so nothing in the corpus records this conflict yet. Sleep never picks the winner of a contradiction: that is a one-way door on stored belief.`,
|
|
12930
|
+
evidence: {
|
|
12931
|
+
kind: "measurement",
|
|
12932
|
+
detail: contradictionEvidence(pending)
|
|
12933
|
+
}
|
|
12934
|
+
});
|
|
12935
|
+
if (outcome === "minted") minted += 1;
|
|
12936
|
+
else if (outcome === "refreshed") refreshed += 1;
|
|
12937
|
+
else if (outcome === "framed") framed += 1;
|
|
12938
|
+
else if (outcome === "dismissed") dismissed += 1;
|
|
12939
|
+
}
|
|
12940
|
+
const closed = judged ? yield* closeVanishedDetections(env, EDGE_REVIEW_DETECTOR, new Set(byKey.keys())) : 0;
|
|
12941
|
+
return {
|
|
12942
|
+
minted,
|
|
12943
|
+
refreshed,
|
|
12944
|
+
framed,
|
|
12945
|
+
dismissed,
|
|
12946
|
+
closed
|
|
12947
|
+
};
|
|
12948
|
+
});
|
|
12949
|
+
/** The canonical finding string: the rel and the two paths, sorted. */
|
|
12950
|
+
const contradictionFinding = (pending) => pending.src < pending.dst ? `contradicts ${pending.src} ${pending.dst}` : `contradicts ${pending.dst} ${pending.src}`;
|
|
12951
|
+
/** The evidence line: the confidence, the detection count, and the model's rationale if it gave one. */
|
|
12952
|
+
const contradictionEvidence = (pending) => `the model judged this a contradiction at confidence ${pending.confidence.toFixed(2)}, detection ${String(pending.detections)} of ${String(2)}` + (pending.rationale === void 0 || pending.rationale.trim() === "" ? "" : `; it said: ${pending.rationale.replace(/\s+/g, " ").trim()}`);
|
|
12953
|
+
/** A path's filename without its extension, for a title that fits `ls` and a commit subject. */
|
|
12954
|
+
const basenameOf = (path) => path.slice(path.lastIndexOf("/") + 1).replace(/\.html$/, "");
|
|
11630
12955
|
|
|
11631
12956
|
//#endregion
|
|
11632
12957
|
//#region packages/sleep/dist/phases/entity-resolution.js
|
|
@@ -11670,6 +12995,15 @@ const edgeTyping = (env) => Effect.gen(function* () {
|
|
|
11670
12995
|
* not cluster, and a cluster below {@link ENTITY_CONFIDENCE_FLOOR}, both land in `reviewCandidates`. An
|
|
11671
12996
|
* entity merge is a one-way door on stored identity: no later commit separates two subjects whose
|
|
11672
12997
|
* memories were fused, and the failure mode of an over-eager gate is silent and permanent.
|
|
12998
|
+
*
|
|
12999
|
+
* **And every one of them now also becomes a TASK.** `reviewCandidates: 2` in a report is issue #44's
|
|
13000
|
+
* motivating example of the failure this phase had: a decision the night deliberately deferred to a
|
|
13001
|
+
* human, reported as a number and then never seen again. A deferred decision IS a task, so
|
|
13002
|
+
* {@link mintReviewTasks} opens one per pair with the band, the score, and each name's file count as
|
|
13003
|
+
* its evidence, keyed so tomorrow refreshes rather than duplicates, and closed when the pair stops
|
|
13004
|
+
* being a candidate — because the pair merging, or the names disappearing, means the question is
|
|
13005
|
+
* answered. The counter survives beside it: the count says how many pairs the night deferred and the
|
|
13006
|
+
* tasks are the ones a human can act on.
|
|
11673
13007
|
*/
|
|
11674
13008
|
/** At or above this ratio two names are the same entity. Auto-merged with no model call. */
|
|
11675
13009
|
const AUTO_MERGE_THRESHOLD = .85;
|
|
@@ -12045,6 +13379,18 @@ const entityResolution = (env) => Effect.gen(function* () {
|
|
|
12045
13379
|
let pendingCorroboration = 0;
|
|
12046
13380
|
let reviewCandidates = 0;
|
|
12047
13381
|
let llmCalls = 0;
|
|
13382
|
+
/** Model calls that came back malformed. The sweep's precondition reads this; see below. */
|
|
13383
|
+
let callsFailed = 0;
|
|
13384
|
+
/**
|
|
13385
|
+
* Every pair this night deferred to a human, as a value rather than only a count.
|
|
13386
|
+
*
|
|
13387
|
+
* This is issue #44's motivating case in one variable. The phase used to report
|
|
13388
|
+
* `reviewCandidates: 2` and the number was never seen again: a decision the night deliberately
|
|
13389
|
+
* declined to make evaporated, and the human it was deferred TO was never told. Keeping the pairs
|
|
13390
|
+
* lets the phase mint one task per pair after the loop, with the evidence that made it a
|
|
13391
|
+
* candidate.
|
|
13392
|
+
*/
|
|
13393
|
+
const deferred = [];
|
|
12048
13394
|
/**
|
|
12049
13395
|
* The model core is skipped entirely on a dry run and when no model is bound, and both leave the
|
|
12050
13396
|
* deterministic passes running. A dry run must make no model call and bump no counter, because a
|
|
@@ -12136,7 +13482,17 @@ const entityResolution = (env) => Effect.gen(function* () {
|
|
|
12136
13482
|
effort: "medium",
|
|
12137
13483
|
toolDescription: "Emit one cluster per subject, naming the members that are the same subject."
|
|
12138
13484
|
});
|
|
12139
|
-
if (clustering === void 0)
|
|
13485
|
+
if (clustering === void 0) {
|
|
13486
|
+
/**
|
|
13487
|
+
* Counted, because the detected-task sweep's precondition reads it. A shard whose call
|
|
13488
|
+
* came back malformed left every one of its names unclustered, so the band pairs among
|
|
13489
|
+
* them are counted as review candidates by the pass below — which is correct for the
|
|
13490
|
+
* REPORT and would be wrong as the sweep's input, since the phase did not actually judge
|
|
13491
|
+
* them. See the sweep's own comment.
|
|
13492
|
+
*/
|
|
13493
|
+
callsFailed += 1;
|
|
13494
|
+
continue;
|
|
13495
|
+
}
|
|
12140
13496
|
for (const cluster of clustering.clusters) {
|
|
12141
13497
|
/**
|
|
12142
13498
|
* A key the batch never offered resolves to nothing, so an invented member cannot become a
|
|
@@ -12171,6 +13527,15 @@ const entityResolution = (env) => Effect.gen(function* () {
|
|
|
12171
13527
|
}
|
|
12172
13528
|
if (cluster.confidence < .7) {
|
|
12173
13529
|
reviewCandidates += 1;
|
|
13530
|
+
deferred.push({
|
|
13531
|
+
entityType,
|
|
13532
|
+
left: merge.alias,
|
|
13533
|
+
right: merge.canonical,
|
|
13534
|
+
reason: "below-floor",
|
|
13535
|
+
score: cluster.confidence,
|
|
13536
|
+
leftFiles: counts.get(merge.alias) ?? 0,
|
|
13537
|
+
rightFiles: counts.get(merge.canonical) ?? 0
|
|
13538
|
+
});
|
|
12174
13539
|
continue;
|
|
12175
13540
|
}
|
|
12176
13541
|
const row = (yield* bumpEntityCorroboration(env.deps.db, {
|
|
@@ -12200,7 +13565,17 @@ const entityResolution = (env) => Effect.gen(function* () {
|
|
|
12200
13565
|
* the decision is recorded either as a merge or as a below-floor review candidate already counted
|
|
12201
13566
|
* above. Counting it here as well would report one pair twice.
|
|
12202
13567
|
*/
|
|
12203
|
-
|
|
13568
|
+
const bandPairs = character.review.filter(([left, right]) => !clusteredPairs.has(pairKey(left, right)));
|
|
13569
|
+
reviewCandidates += bandPairs.length;
|
|
13570
|
+
for (const [left, right] of bandPairs) deferred.push({
|
|
13571
|
+
entityType,
|
|
13572
|
+
left,
|
|
13573
|
+
right,
|
|
13574
|
+
reason: "character-band",
|
|
13575
|
+
score: nameSimilarity(left, right),
|
|
13576
|
+
leftFiles: counts.get(left) ?? 0,
|
|
13577
|
+
rightFiles: counts.get(right) ?? 0
|
|
13578
|
+
});
|
|
12204
13579
|
/** One union-find over every accepted pair, so the three sources cannot disagree on a root. */
|
|
12205
13580
|
const aliasToCanonical = unionPairs(counts, accepted);
|
|
12206
13581
|
for (const entity of bucket) {
|
|
@@ -12220,12 +13595,32 @@ const entityResolution = (env) => Effect.gen(function* () {
|
|
|
12220
13595
|
aliasMerges,
|
|
12221
13596
|
pendingCorroboration,
|
|
12222
13597
|
reviewCandidates,
|
|
13598
|
+
tasksMinted: 0,
|
|
13599
|
+
tasksFramed: 0,
|
|
13600
|
+
tasksDismissed: 0,
|
|
13601
|
+
tasksClosed: 0,
|
|
12223
13602
|
filesRewritten: rewrites.size
|
|
12224
13603
|
};
|
|
12225
|
-
|
|
13604
|
+
/**
|
|
13605
|
+
* A dry run stops here and mints nothing, matching what the rest of this phase already declines
|
|
13606
|
+
* to do on one. `reviewCandidates` is real on a dry run; the tasks it would open are not.
|
|
13607
|
+
*/
|
|
13608
|
+
if (env.dryRun) return {
|
|
12226
13609
|
...emptyOutcome(counts),
|
|
12227
13610
|
llmCalls
|
|
12228
13611
|
};
|
|
13612
|
+
/**
|
|
13613
|
+
* The deferred decisions become task files, keyed and capped, in the SAME commit as the merges.
|
|
13614
|
+
*
|
|
13615
|
+
* One commit rather than two, because the two halves are one night's answer to the same question:
|
|
13616
|
+
* these pairs merged, those the phase declined to merge and handed to you. A reviewer reads the
|
|
13617
|
+
* pair together, and `commitPhase` commits whatever is staged, so the mints ride along.
|
|
13618
|
+
*
|
|
13619
|
+
* Mints happen even when nothing was rewritten, and that reordering is the whole point of surface
|
|
13620
|
+
* 1. The old early return on `rewrites.size === 0` would have skipped exactly the night this
|
|
13621
|
+
* feature exists for: a night whose only outcome was deferrals is a night with no rewrites.
|
|
13622
|
+
*/
|
|
13623
|
+
const tasks = yield* mintReviewTasks(env, deferred, model !== void 0 && callsFailed === 0);
|
|
12229
13624
|
let rewritten = 0;
|
|
12230
13625
|
for (const [path, pairs] of [...rewrites.entries()].sort(([left], [right]) => left < right ? -1 : 1)) {
|
|
12231
13626
|
const html = yield* readFileBytes(env, path);
|
|
@@ -12240,14 +13635,113 @@ const entityResolution = (env) => Effect.gen(function* () {
|
|
|
12240
13635
|
}
|
|
12241
13636
|
const final = {
|
|
12242
13637
|
...counts,
|
|
13638
|
+
tasksMinted: tasks.minted,
|
|
13639
|
+
tasksFramed: tasks.framed,
|
|
13640
|
+
tasksDismissed: tasks.dismissed,
|
|
13641
|
+
tasksClosed: tasks.closed,
|
|
12243
13642
|
filesRewritten: rewritten
|
|
12244
13643
|
};
|
|
13644
|
+
/**
|
|
13645
|
+
* Nothing staged, no commit. `commitPhase` already no-ops on an empty index, so this only spares
|
|
13646
|
+
* git the call — and it now has to consider the MINTS as well as the rewrites, because a night
|
|
13647
|
+
* whose only output is deferred-decision tasks must still commit them.
|
|
13648
|
+
*/
|
|
13649
|
+
if (rewritten === 0 && tasks.minted === 0 && tasks.refreshed === 0 && tasks.closed === 0) return {
|
|
13650
|
+
...emptyOutcome(final),
|
|
13651
|
+
llmCalls
|
|
13652
|
+
};
|
|
12245
13653
|
return {
|
|
12246
13654
|
counts: final,
|
|
12247
|
-
commitSha: yield* commitPhase(env, "entity-resolution", `normalize ${normalized} entity names, merge ${fuzzyMerges} aliases`, final),
|
|
13655
|
+
commitSha: yield* commitPhase(env, "entity-resolution", `normalize ${normalized} entity names, merge ${fuzzyMerges} aliases`, final, tasks.minted + tasks.closed === 0 ? void 0 : `deferred ${tasks.minted} alias decisions to review tasks` + (tasks.closed === 0 ? "" : `; closed ${tasks.closed}: no longer detected`)),
|
|
12248
13656
|
llmCalls
|
|
12249
13657
|
};
|
|
12250
13658
|
});
|
|
13659
|
+
/** The detector name every alias review task is keyed and swept under. */
|
|
13660
|
+
const ENTITY_REVIEW_DETECTOR = "entity-resolution";
|
|
13661
|
+
/**
|
|
13662
|
+
* Mint one review task per deferred pair, and sweep the ones that stopped being deferred.
|
|
13663
|
+
*
|
|
13664
|
+
* **The key is the entity TYPE plus the two names sorted**, and not the reason. A pair the character
|
|
13665
|
+
* band deferred last night and the model deferred below the floor tonight is ONE question a human has
|
|
13666
|
+
* to answer once — are these the same subject — so it must key the same however the night arrived at
|
|
13667
|
+
* it. Sorting is what makes `(laith, laith al-saadoon)` and the reverse one key; the pair is
|
|
13668
|
+
* unordered, because neither name is the subject of the question.
|
|
13669
|
+
*
|
|
13670
|
+
* **The evidence is a MEASUREMENT and says so.** There is no sentence anywhere in the corpus stating
|
|
13671
|
+
* that two names scored 0.79 against each other, so a quote would have to be manufactured. The
|
|
13672
|
+
* `DetectionEvidence` union makes that difference explicit rather than leaving it to a convention this
|
|
13673
|
+
* function could quietly break.
|
|
13674
|
+
*
|
|
13675
|
+
* **The sweep is gated on a night that had a MODEL and lost no call**, which `judged` carries.
|
|
13676
|
+
* `deferred` holds what the phase actually decided to defer, and a shard whose model call failed left
|
|
13677
|
+
* its names unclustered — so its band pairs are reported as review candidates without having been
|
|
13678
|
+
* judged. They ARE still live, so they belong in `liveKeys`; but a night that lost a call cannot
|
|
13679
|
+
* distinguish "the model decided this pair is fine" from "the model was never asked", and closing on
|
|
13680
|
+
* that reading would take a real review out of a human's queue because Bedrock throttled.
|
|
13681
|
+
*
|
|
13682
|
+
* **`callsFailed === 0` alone was the bug, because it is VACUOUSLY TRUE with no model bound.** The
|
|
13683
|
+
* caller now requires `model !== undefined` as well. A credential-free night runs only the two
|
|
13684
|
+
* deterministic passes, so it produces `character-band` deferrals and CANNOT produce a `below-floor`
|
|
13685
|
+
* one — a below-floor deferral is by definition a merge the model proposed under
|
|
13686
|
+
* `ENTITY_CONFIDENCE_FLOOR`, and there was no model to propose it. Its `deferred` therefore omits every
|
|
13687
|
+
* below-floor pair a model night opened, and sweeping against that closed those tasks on the first
|
|
13688
|
+
* night without credentials. `tasks.ts`'s `closeVanishedDetections` states this precondition as
|
|
13689
|
+
* "a phase that degraded — no model bound, a batch whose call failed — did not evaluate the candidate
|
|
13690
|
+
* set", and no-model is the arm that check had missed.
|
|
13691
|
+
*/
|
|
13692
|
+
const mintReviewTasks = (env, deferred, judged) => Effect.gen(function* () {
|
|
13693
|
+
const budget = budgetFor(env);
|
|
13694
|
+
/**
|
|
13695
|
+
* Sorted and de-duplicated by key before minting, so the order tasks are opened in is a function
|
|
13696
|
+
* of the pairs and not of which entity type happened to be walked first — which matters once the
|
|
13697
|
+
* budget bites, because then the ORDER decides which pairs a human sees.
|
|
13698
|
+
*/
|
|
13699
|
+
const byKey = /* @__PURE__ */ new Map();
|
|
13700
|
+
for (const candidate of [...deferred].sort(compareCandidates)) {
|
|
13701
|
+
const key = detectionKey(ENTITY_REVIEW_DETECTOR, findingFor(candidate));
|
|
13702
|
+
if (!byKey.has(key)) byKey.set(key, candidate);
|
|
13703
|
+
}
|
|
13704
|
+
let minted = 0;
|
|
13705
|
+
let refreshed = 0;
|
|
13706
|
+
let framed = 0;
|
|
13707
|
+
let dismissed = 0;
|
|
13708
|
+
for (const candidate of byKey.values()) {
|
|
13709
|
+
const outcome = yield* mintDetectedTask(env, budget, {
|
|
13710
|
+
detector: ENTITY_REVIEW_DETECTOR,
|
|
13711
|
+
finding: findingFor(candidate),
|
|
13712
|
+
title: `Confirm whether ${candidate.left} and ${candidate.right} are one ${candidate.entityType}`,
|
|
13713
|
+
claim: `confirm: are "${candidate.left}" and "${candidate.right}" the same ${candidate.entityType}?`,
|
|
13714
|
+
detail: "Sleep declined to merge them and deferred the decision. Merging two entities is a one-way door: no later commit separates two subjects whose memories were fused.",
|
|
13715
|
+
evidence: {
|
|
13716
|
+
kind: "measurement",
|
|
13717
|
+
detail: evidenceFor(candidate)
|
|
13718
|
+
}
|
|
13719
|
+
});
|
|
13720
|
+
if (outcome === "minted") minted += 1;
|
|
13721
|
+
else if (outcome === "refreshed") refreshed += 1;
|
|
13722
|
+
else if (outcome === "framed") framed += 1;
|
|
13723
|
+
else if (outcome === "dismissed") dismissed += 1;
|
|
13724
|
+
}
|
|
13725
|
+
const closed = judged ? yield* closeVanishedDetections(env, ENTITY_REVIEW_DETECTOR, new Set(byKey.keys())) : 0;
|
|
13726
|
+
return {
|
|
13727
|
+
minted,
|
|
13728
|
+
refreshed,
|
|
13729
|
+
framed,
|
|
13730
|
+
dismissed,
|
|
13731
|
+
closed
|
|
13732
|
+
};
|
|
13733
|
+
});
|
|
13734
|
+
/** The canonical finding string: the type and the two names, sorted. See {@link mintReviewTasks}. */
|
|
13735
|
+
const findingFor = (candidate) => candidate.left < candidate.right ? `${candidate.entityType} ${candidate.left} ${candidate.right}` : `${candidate.entityType} ${candidate.right} ${candidate.left}`;
|
|
13736
|
+
/** The evidence line: which band deferred it, at what number, and how much corpus is behind each name. */
|
|
13737
|
+
const evidenceFor = (candidate) => (candidate.reason === "character-band" ? `character overlap ${candidate.score.toFixed(2)}, inside the ${String(REVIEW_THRESHOLD)}-${String(AUTO_MERGE_THRESHOLD)} review band` : `the model proposed the merge at confidence ${candidate.score.toFixed(2)}, below the ${String(ENTITY_CONFIDENCE_FLOOR)} floor`) + `; "${candidate.left}" is claimed by ${String(candidate.leftFiles)} active memories and "${candidate.right}" by ${String(candidate.rightFiles)}`;
|
|
13738
|
+
/** Type, then the two names, then the reason. A total order, so the mint sequence is reproducible. */
|
|
13739
|
+
const compareCandidates = (left, right) => {
|
|
13740
|
+
const leftFinding = findingFor(left);
|
|
13741
|
+
const rightFinding = findingFor(right);
|
|
13742
|
+
if (leftFinding !== rightFinding) return leftFinding < rightFinding ? -1 : 1;
|
|
13743
|
+
return left.reason < right.reason ? -1 : left.reason > right.reason ? 1 : 0;
|
|
13744
|
+
};
|
|
12251
13745
|
|
|
12252
13746
|
//#endregion
|
|
12253
13747
|
//#region packages/sleep/dist/publish.js
|
|
@@ -12988,6 +14482,275 @@ const parseSidecar = (contents) => {
|
|
|
12988
14482
|
};
|
|
12989
14483
|
const numberOr = (value, fallback) => typeof value === "number" && Number.isFinite(value) ? value : fallback;
|
|
12990
14484
|
|
|
14485
|
+
//#endregion
|
|
14486
|
+
//#region packages/sleep/dist/phases/task-detection.js
|
|
14487
|
+
/**
|
|
14488
|
+
* Phase 13, task detection. A batched scan over the recent active corpus for work the text records
|
|
14489
|
+
* and nobody opened. ONE commit for the night's mints.
|
|
14490
|
+
*
|
|
14491
|
+
* Surface 3 of issue #44, and the only one that is net-new model spend. Surfaces 1 and 2 ride on
|
|
14492
|
+
* decisions other phases were already making — a review band entity resolution declined, a pair the
|
|
14493
|
+
* divergence veto refused, a commitment the consolidator's existing call can also report — so they
|
|
14494
|
+
* cost tokens the night was already spending. This one asks a question nobody was asking, which is why
|
|
14495
|
+
* it is capped, floored, and last of the four: the issue explicitly sizes it as the surface that could
|
|
14496
|
+
* ship last or never.
|
|
14497
|
+
*
|
|
14498
|
+
* Four stages, and the separation is what keeps a model's sentence from becoming an assertion:
|
|
14499
|
+
*
|
|
14500
|
+
* 1. **Scan (SQL, no model).** {@link TASK_SCAN_LIMIT} most-recently-updated active non-task
|
|
14501
|
+
* memories, newest first, ties by path. Deterministic, so the batches and the `m1`..`mN` keys are a
|
|
14502
|
+
* function of the corpus.
|
|
14503
|
+
* 2. **Batch (deterministic).** Sliced at {@link TASK_DETECT_BATCH_SIZE} on the shared kernel, each
|
|
14504
|
+
* member cut to {@link TASK_DETECT_MEMBER_CHARS}. One call per batch, never one per memory: 200
|
|
14505
|
+
* candidates is 10 calls, and per-memory judging is 200.
|
|
14506
|
+
* 3. **Ask (one isolated call per batch).** A failure skips its batch and is counted. A night that
|
|
14507
|
+
* scanned nine batches and lost the tenth has done nine batches of work.
|
|
14508
|
+
* 4. **Mint (deterministic, and this is where the guards are).** The key must resolve to an offered
|
|
14509
|
+
* member; the confidence must clear {@link TASK_DETECT_FLOOR}; the sentence must exist VERBATIM in
|
|
14510
|
+
* the cited file's own article text, which `mintDetectedTask` checks by reading the file; the
|
|
14511
|
+
* nightly budget must have room. Everything the file says is derived from the member the model was
|
|
14512
|
+
* shown plus its own quoted sentence — the model never names a path, a title, or a status.
|
|
14513
|
+
*
|
|
14514
|
+
* **No self-referential loops, and ONE guard rather than two.** `recentActiveMemories` excludes `task`
|
|
14515
|
+
* in SQL, and that is the whole mechanism: a task is not evidence of another task, and a detector that
|
|
14516
|
+
* scanned its own output would restate its own queue every night. A path-prefix check on top of it was
|
|
14517
|
+
* written and then removed, because it could not be made to fire — the index is refreshed once in
|
|
14518
|
+
* preflight, so a task minted earlier in the same night is ABSENT from the projection rather than
|
|
14519
|
+
* present with the wrong type, and either way the statement does not return it. Mutation-verified:
|
|
14520
|
+
* deleting the SQL filter fails `tests/task-detection.test.ts`, and deleting the path filter did not.
|
|
14521
|
+
*
|
|
14522
|
+
* **Self-cleaning, and only from a full-strength scan.** {@link closeVanishedDetections} runs when the
|
|
14523
|
+
* night reached every batch — no skips — because `liveKeys` then genuinely describes the findings that
|
|
14524
|
+
* still exist. On a night that lost a batch it describes what the phase managed to look at, and
|
|
14525
|
+
* sweeping against that would close a human's review because a call was throttled.
|
|
14526
|
+
*
|
|
14527
|
+
* **Degrades and never fails.** No model bound, no candidate, a dry run, or a night where nothing
|
|
14528
|
+
* clears the floor all produce `ok` with counts. A credential-free run is not a broken run.
|
|
14529
|
+
*/
|
|
14530
|
+
/**
|
|
14531
|
+
* Memories scanned per night.
|
|
14532
|
+
*
|
|
14533
|
+
* 200, matching `EDGE_TYPING_CANDIDATE_LIMIT`'s posture rather than `COMPRESS_CANDIDATE_LIMIT`'s: this
|
|
14534
|
+
* is a bound on what the phase READS INTO PROMPTS, and every candidate costs tokens whether or not it
|
|
14535
|
+
* yields a finding. At {@link TASK_DETECT_BATCH_SIZE} that is ten calls a night, which sits inside the
|
|
14536
|
+
* envelope issue #43 measured for the whole batching direction. Newest-first ordering is what makes
|
|
14537
|
+
* 200 a moving window rather than a truncation: a corpus of 2,907 is scanned in the region where
|
|
14538
|
+
* unresolved work actually lives, and last month's settled memories are not re-read every night.
|
|
14539
|
+
*/
|
|
14540
|
+
const TASK_SCAN_LIMIT = 200;
|
|
14541
|
+
/**
|
|
14542
|
+
* Memories offered per model call.
|
|
14543
|
+
*
|
|
14544
|
+
* Twenty. The question is per member and the answer is a short list, so the batch can be wider than
|
|
14545
|
+
* compress's 8 (which has to hold every member's facts in the answer's generative attention) and
|
|
14546
|
+
* narrower than dedup's 40 (whose members are pre-grouped, so most of a batch needs no independent
|
|
14547
|
+
* judgment). Twenty memories at {@link TASK_DETECT_MEMBER_CHARS} is 24k characters of member text, and
|
|
14548
|
+
* the model has to read each one for a distinct verbatim sentence.
|
|
14549
|
+
*/
|
|
14550
|
+
const TASK_DETECT_BATCH_SIZE = 20;
|
|
14551
|
+
/** Characters of each member shown. The house per-member budget, the same 1200 four phases use. */
|
|
14552
|
+
const TASK_DETECT_MEMBER_CHARS = 1200;
|
|
14553
|
+
/**
|
|
14554
|
+
* The confidence a finding must clear before a task is minted.
|
|
14555
|
+
*
|
|
14556
|
+
* 0.7, the same floor `EDGE_CONFIDENCE_FLOOR` and `ENTITY_CONFIDENCE_FLOOR` set, and for the
|
|
14557
|
+
* comparable reason: a false positive costs a reviewer's attention, which is the resource this whole
|
|
14558
|
+
* surface spends, and the failure mode of a low floor is a queue nobody reads. One number rather than
|
|
14559
|
+
* one per kind, because a second would be a knob nobody could state the meaning of.
|
|
14560
|
+
*/
|
|
14561
|
+
const TASK_DETECT_FLOOR = .7;
|
|
14562
|
+
/** The detector's name: the key's namespace, the task's second tag, and the sweep's scope. */
|
|
14563
|
+
const TASK_DETECT_DETECTOR = "task-detection";
|
|
14564
|
+
/** The text a member is offered under: its title, claim, and body, the join compress and dedup use. */
|
|
14565
|
+
const memberText = (row) => `${row.title}\n${row.gist}\n${row.body_text}`;
|
|
14566
|
+
/**
|
|
14567
|
+
* The claim a finding becomes, by kind.
|
|
14568
|
+
*
|
|
14569
|
+
* Derived here and never asked of the model, the same decision every other phase makes about a value
|
|
14570
|
+
* that reaches a file. A model-written claim would be the `<mark>` span, `files.gist`, and the frame
|
|
14571
|
+
* key the proximity check reads — so the one sentence that decides how this task is de-duplicated
|
|
14572
|
+
* against the rest of the queue would be prose a model chose.
|
|
14573
|
+
*
|
|
14574
|
+
* The verbs are the imperative a reviewer acts on: a commitment is confirmed or closed, a follow-up is
|
|
14575
|
+
* resolved or dismissed. Both name the SOURCE, because a task whose subject a reader has to go
|
|
14576
|
+
* looking for is a task they skip.
|
|
14577
|
+
*/
|
|
14578
|
+
const claimFor = (kind, path) => kind === "commitment" ? `confirm: ${path} records a commitment with nothing saying it was done.` : `resolve: ${path} leaves a follow-up open.`;
|
|
14579
|
+
/** The title a finding becomes. Same two shapes, without the trailing sentence punctuation. */
|
|
14580
|
+
const titleFor$1 = (kind, row) => kind === "commitment" ? `Confirm the commitment recorded in ${row.title}` : `Resolve the follow-up left open by ${row.title}`;
|
|
14581
|
+
const taskDetection = (env) => Effect.gen(function* () {
|
|
14582
|
+
const model = env.deps.model;
|
|
14583
|
+
if (model === void 0) return {
|
|
14584
|
+
...emptyOutcome(ZERO),
|
|
14585
|
+
detail: "no model bound"
|
|
14586
|
+
};
|
|
14587
|
+
/**
|
|
14588
|
+
* The candidate slice, with the self-scan exclusion inside the statement. See the phase header:
|
|
14589
|
+
* `recentActiveMemories` filters `memory_type` in SQL, and no second path-level filter is added
|
|
14590
|
+
* here on purpose — a detected task's row either carries `memory_type = 'task'` and the statement
|
|
14591
|
+
* excludes it, or is absent from the index entirely and the statement never sees it. A path-prefix
|
|
14592
|
+
* check would be a guard with no reachable input, which is worse than no guard: it reads as the
|
|
14593
|
+
* thing standing between a task and the prompt while the statement is what actually does it.
|
|
14594
|
+
*/
|
|
14595
|
+
const candidates = yield* recentActiveMemories(env.deps.db, { limit: 200 });
|
|
14596
|
+
if (candidates.length === 0) return emptyOutcome(ZERO);
|
|
14597
|
+
/**
|
|
14598
|
+
* A dry run stops after the deterministic half, before the calls. The candidate count is the
|
|
14599
|
+
* number an operator sizing a night wants, and a preview that spent the tokens to then discard
|
|
14600
|
+
* every answer would be the most expensive way to produce it. `entity-resolution` makes the same
|
|
14601
|
+
* choice for a stronger reason (its dry run would have to manufacture a night of corroboration);
|
|
14602
|
+
* here it is simply that nothing the calls buy survives a dry run.
|
|
14603
|
+
*/
|
|
14604
|
+
if (env.dryRun) return emptyOutcome({
|
|
14605
|
+
...ZERO,
|
|
14606
|
+
candidates: candidates.length
|
|
14607
|
+
});
|
|
14608
|
+
const batches = assembleBatches([candidates], { maxMembers: 20 });
|
|
14609
|
+
const modelKey = modelFor(env.deps, "task-detection");
|
|
14610
|
+
const budget = budgetFor(env);
|
|
14611
|
+
let llmCalls = 0;
|
|
14612
|
+
let findings = 0;
|
|
14613
|
+
let minted = 0;
|
|
14614
|
+
let refreshed = 0;
|
|
14615
|
+
let unverified = 0;
|
|
14616
|
+
let framed = 0;
|
|
14617
|
+
let dismissed = 0;
|
|
14618
|
+
let skipped = 0;
|
|
14619
|
+
/**
|
|
14620
|
+
* Every key this night's scan SAW, whether or not it minted and whether or not it cleared the floor.
|
|
14621
|
+
* The sweep's input; see the `liveKeys.add` below for why the floor is not a filter here.
|
|
14622
|
+
*/
|
|
14623
|
+
const liveKeys = /* @__PURE__ */ new Set();
|
|
14624
|
+
for (const batch of batches) {
|
|
14625
|
+
const keyed = keyMembers(batch, memberText, { charBudget: TASK_DETECT_MEMBER_CHARS });
|
|
14626
|
+
llmCalls += 1;
|
|
14627
|
+
const answer = yield* batchCall(model, `task-detection batch of ${batch.length}`, {
|
|
14628
|
+
schema: TaskDetection,
|
|
14629
|
+
system: TASK_DETECT_SYSTEM,
|
|
14630
|
+
prompt: taskDetectPrompt(keyed.keyed),
|
|
14631
|
+
modelKey,
|
|
14632
|
+
effort: "medium",
|
|
14633
|
+
toolDescription: "Emit one finding per memory that records open work, quoting the sentence verbatim."
|
|
14634
|
+
});
|
|
14635
|
+
if (answer === void 0) {
|
|
14636
|
+
skipped += 1;
|
|
14637
|
+
continue;
|
|
14638
|
+
}
|
|
14639
|
+
/**
|
|
14640
|
+
* The keys this batch has already yielded a finding for, so a SECOND finding naming one is
|
|
14641
|
+
* dropped. Same guard `edge-typing` carries and the same reason: nothing in the schema stops a
|
|
14642
|
+
* model from emitting two findings for one member, and acting on both would mint two tasks about
|
|
14643
|
+
* one memory whose only difference is which sentence was quoted. `resolveKeys` does not help,
|
|
14644
|
+
* because it is called one key at a time here — a finding names one member.
|
|
14645
|
+
*/
|
|
14646
|
+
const answered = /* @__PURE__ */ new Set();
|
|
14647
|
+
for (const finding of answer.findings) {
|
|
14648
|
+
const [row] = resolveKeys(keyed, [finding.memberKey]);
|
|
14649
|
+
if (row === void 0) continue;
|
|
14650
|
+
if (answered.has(finding.memberKey)) continue;
|
|
14651
|
+
answered.add(finding.memberKey);
|
|
14652
|
+
findings += 1;
|
|
14653
|
+
/**
|
|
14654
|
+
* The key is the SOURCE PATH plus the normalized sentence, so the same commitment found again
|
|
14655
|
+
* tomorrow keys the same and refreshes. The path is in the key rather than only the sentence
|
|
14656
|
+
* because one sentence can legitimately appear in two memories — a corrected memory and its
|
|
14657
|
+
* correction share most of their prose — and those are two findings a reviewer decides
|
|
14658
|
+
* separately. `detectionKey` normalizes, so a member whose whitespace the chunker changed keys
|
|
14659
|
+
* the same.
|
|
14660
|
+
*/
|
|
14661
|
+
const key = detectionKey(TASK_DETECT_DETECTOR, `${row.path} ${finding.sentence}`);
|
|
14662
|
+
/**
|
|
14663
|
+
* The key is live BEFORE the floor gate, which is `closeVanishedDetections`' contract verbatim:
|
|
14664
|
+
* `liveKeys` is every finding the detector SAW, not every finding it minted.
|
|
14665
|
+
*
|
|
14666
|
+
* A below-floor finding was SEEN. The sentence is still in the file and the model still reported
|
|
14667
|
+
* it; only the confidence moved. Adding the key after the gate makes a task's life a function of
|
|
14668
|
+
* confidence JITTER across nights — minted at 0.72, swept at 0.68, re-minted at 0.71 — and the
|
|
14669
|
+
* sweep archives, so each cycle takes the file out of the human's directory and back into it
|
|
14670
|
+
* with a fresh `memhtml-created`. The finding VANISHING is what closure is for, and a confidence
|
|
14671
|
+
* that dipped one hundredth is not that.
|
|
14672
|
+
*
|
|
14673
|
+
* A below-floor finding therefore keeps its task open without ever being able to open one, which
|
|
14674
|
+
* is the asymmetry the floor is supposed to buy: the floor guards what enters a human's queue,
|
|
14675
|
+
* not what stays there once they have been shown it.
|
|
14676
|
+
*/
|
|
14677
|
+
liveKeys.add(key);
|
|
14678
|
+
if (finding.confidence < .7) continue;
|
|
14679
|
+
const outcome = yield* mintDetectedTask(env, budget, {
|
|
14680
|
+
detector: TASK_DETECT_DETECTOR,
|
|
14681
|
+
finding: `${row.path} ${finding.sentence}`,
|
|
14682
|
+
title: titleFor$1(finding.kind, row),
|
|
14683
|
+
claim: claimFor(finding.kind, row.path),
|
|
14684
|
+
detail: `Detected as ${finding.kind === "commitment" ? "a commitment" : "an unresolved follow-up"} at confidence ${finding.confidence.toFixed(2)} in a ${row.memory_type} memory last updated ${row.updated_at}.`,
|
|
14685
|
+
evidence: {
|
|
14686
|
+
kind: "quote",
|
|
14687
|
+
quote: finding.sentence,
|
|
14688
|
+
sourcePath: row.path
|
|
14689
|
+
}
|
|
14690
|
+
});
|
|
14691
|
+
if (outcome === "minted") minted += 1;
|
|
14692
|
+
else if (outcome === "refreshed") refreshed += 1;
|
|
14693
|
+
else if (outcome === "unverified") unverified += 1;
|
|
14694
|
+
else if (outcome === "framed") framed += 1;
|
|
14695
|
+
else if (outcome === "dismissed") dismissed += 1;
|
|
14696
|
+
}
|
|
14697
|
+
}
|
|
14698
|
+
/**
|
|
14699
|
+
* The sweep, only from a full-strength scan. `skipped > 0` means at least one batch's memories
|
|
14700
|
+
* went unread, so a finding of theirs is missing from `liveKeys` because the phase could not look
|
|
14701
|
+
* rather than because it is gone.
|
|
14702
|
+
*/
|
|
14703
|
+
const closed = skipped === 0 ? yield* closeVanishedDetections(env, TASK_DETECT_DETECTOR, liveKeys) : 0;
|
|
14704
|
+
const counts = {
|
|
14705
|
+
candidates: candidates.length,
|
|
14706
|
+
batches: batches.length,
|
|
14707
|
+
findings,
|
|
14708
|
+
minted,
|
|
14709
|
+
refreshed,
|
|
14710
|
+
unverified,
|
|
14711
|
+
framed,
|
|
14712
|
+
dismissed,
|
|
14713
|
+
closed,
|
|
14714
|
+
capped: budget.overflow,
|
|
14715
|
+
skipped
|
|
14716
|
+
};
|
|
14717
|
+
/**
|
|
14718
|
+
* A refresh writes a `memhtml-updated` stamp, which is a staged file, so it commits — the queue's
|
|
14719
|
+
* "last seen" is a fact worth a diff. Nothing staged at all leaves `commitSha: null`, which
|
|
14720
|
+
* `commitPhase` already produces on an empty index; the early return only spares git the call.
|
|
14721
|
+
*/
|
|
14722
|
+
if (minted === 0 && refreshed === 0 && closed === 0) return {
|
|
14723
|
+
counts,
|
|
14724
|
+
commitSha: null,
|
|
14725
|
+
llmCalls
|
|
14726
|
+
};
|
|
14727
|
+
return {
|
|
14728
|
+
counts,
|
|
14729
|
+
commitSha: yield* commitPhase(env, "task-detection", `open ${minted} detected tasks, close ${closed} no longer detected`, counts, closed === 0 ? void 0 : "closing reason: no longer detected"),
|
|
14730
|
+
llmCalls
|
|
14731
|
+
};
|
|
14732
|
+
});
|
|
14733
|
+
/**
|
|
14734
|
+
* The full count SHAPE, at zero.
|
|
14735
|
+
*
|
|
14736
|
+
* Every key the phase can report is present on every path, because a report reader comparing two
|
|
14737
|
+
* nights reads a missing key as a phase that does not have that concept rather than as a night that
|
|
14738
|
+
* did none of it. Same rule `edge-typing`'s `zero` states.
|
|
14739
|
+
*/
|
|
14740
|
+
const ZERO = {
|
|
14741
|
+
candidates: 0,
|
|
14742
|
+
batches: 0,
|
|
14743
|
+
findings: 0,
|
|
14744
|
+
minted: 0,
|
|
14745
|
+
refreshed: 0,
|
|
14746
|
+
unverified: 0,
|
|
14747
|
+
framed: 0,
|
|
14748
|
+
dismissed: 0,
|
|
14749
|
+
closed: 0,
|
|
14750
|
+
capped: 0,
|
|
14751
|
+
skipped: 0
|
|
14752
|
+
};
|
|
14753
|
+
|
|
12991
14754
|
//#endregion
|
|
12992
14755
|
//#region packages/sleep/dist/phases/trace-consolidation.js
|
|
12993
14756
|
/**
|
|
@@ -13029,6 +14792,48 @@ const numberOr = (value, fallback) => typeof value === "number" && Number.isFini
|
|
|
13029
14792
|
* (missing credentials, an unreachable agent, an off-contract answer), and a candidate this phase
|
|
13030
14793
|
* refuses all produce `ok` with counts and a reason. INV-3 in full: a night with no Bedrock
|
|
13031
14794
|
* credentials is not a broken night, and a run that lost this phase stays green.
|
|
14795
|
+
*
|
|
14796
|
+
* ## Surface 2: the same answer also carries COMMITMENTS
|
|
14797
|
+
*
|
|
14798
|
+
* The consolidator's turn now reports two lists, and the second is issue #44's surface 2. The marginal
|
|
14799
|
+
* cost is tokens in a call this phase was already making — no new model call, which is what makes this
|
|
14800
|
+
* surface cheap enough to run every night and is the reason the issue sizes it above the net-new scan.
|
|
14801
|
+
*
|
|
14802
|
+
* A commitment is not a candidate memory and does not travel through the candidate loop.
|
|
14803
|
+
* {@link CONSOLIDATION_KINDS} excludes `task` deliberately ("task is work to do, not something observed
|
|
14804
|
+
* to have happened"), and that exclusion still holds: the model reports what a transcript SAYS, and the
|
|
14805
|
+
* decision to open a task is made HERE, deterministically, by {@link commitmentRefusalFor} plus
|
|
14806
|
+
* {@link COMMITMENT_FLOOR}.
|
|
14807
|
+
*
|
|
14808
|
+
* Two arms, from one list:
|
|
14809
|
+
*
|
|
14810
|
+
* - **Unresolved** commitments mint detected tasks, sharing the night's `DETECTED_TASK_CAP` budget with
|
|
14811
|
+
* every other detector, keyed on a normalized digest of the STATEMENT so the same promise restated on
|
|
14812
|
+
* a later night refreshes rather than duplicating.
|
|
14813
|
+
* - **Resolved** commitments — a session showing the work done — close an OPEN detected task whose key
|
|
14814
|
+
* matches. That is the issue's "closure is also detected", and it is the reason a commitment that
|
|
14815
|
+
* arrives already-done is still worth reporting: a night that opens a task and a later night that
|
|
14816
|
+
* closes it are two readings of the same commitment.
|
|
14817
|
+
*
|
|
14818
|
+
* The key carries the statement and NOT the session, which is the one place surface 2 departs from
|
|
14819
|
+
* `task-detection`'s keying, and it is forced by what closure has to reach across. See
|
|
14820
|
+
* {@link commitmentKey}.
|
|
14821
|
+
*
|
|
14822
|
+
* **Only a DETECTED task is ever closed, and the guard is `closeDetectedTask`'s, on the path.** A
|
|
14823
|
+
* human-opened task must not be archived because a model read "shipped it" in somebody's scrollback.
|
|
14824
|
+
*
|
|
14825
|
+
* **A commitment's evidence quote never enters the corpus, exactly like a candidate's.** The task body
|
|
14826
|
+
* carries the model's own restatement plus the session id as a `memhtml-session` stamp; the verbatim
|
|
14827
|
+
* line goes in the commit message. `packages/sleep/src/tasks.ts`' `DetectionEvidence` `session` arm is
|
|
14828
|
+
* where that split is enforced, and its header records why the quote is not re-verified against
|
|
14829
|
+
* transcript bytes.
|
|
14830
|
+
*
|
|
14831
|
+
* **One commit for the batch of commitment tasks**, not one per task, and that is the one place this
|
|
14832
|
+
* phase departs from its one-commit-per-candidate discipline. The reason the discipline exists is that
|
|
14833
|
+
* a distilled memory is a standalone ASSERTION about the world a reviewer weighs on its own. A detected
|
|
14834
|
+
* task asserts nothing — it is a proposal, and the reviewer's decision is made in the task file rather
|
|
14835
|
+
* than at the commit. What the commit has to do is be reviewable, and "the night found four
|
|
14836
|
+
* commitments, here they are with their quotes" is one reviewable decision about one model answer.
|
|
13032
14837
|
*/
|
|
13033
14838
|
/**
|
|
13034
14839
|
* The smallest transcript worth a model's attention, in bytes.
|
|
@@ -13076,6 +14881,270 @@ const COMMIT_QUOTE_CHARS = 200;
|
|
|
13076
14881
|
/** Where a consolidated memory lands: by kind and tag, exactly as an agent's own write is placed. */
|
|
13077
14882
|
const CONSOLIDATION_TAG = "trace-consolidation";
|
|
13078
14883
|
/**
|
|
14884
|
+
* The confidence a commitment must clear before it mints a task or closes one.
|
|
14885
|
+
*
|
|
14886
|
+
* 0.7, the same floor `TASK_DETECT_FLOOR`, `EDGE_CONFIDENCE_FLOOR`, and `ENTITY_CONFIDENCE_FLOOR` set,
|
|
14887
|
+
* and one number rather than one per arm. The mint arm and the closure arm read it identically on
|
|
14888
|
+
* purpose: they are the same judgement about the same sentence, made once, and a lower floor on closure
|
|
14889
|
+
* would mean a commitment too weak to open a task was strong enough to close one.
|
|
14890
|
+
*
|
|
14891
|
+
* The resource this bounds is a reviewer's attention, which is a property of the human rather than of
|
|
14892
|
+
* how the finding was reached — the reasoning `DETECTED_TASK_CAP` records for being shared.
|
|
14893
|
+
*/
|
|
14894
|
+
const COMMITMENT_FLOOR = .7;
|
|
14895
|
+
/** The detector name every commitment task is keyed, tagged, and closed under. */
|
|
14896
|
+
const COMMITMENT_DETECTOR = "trace-commitment";
|
|
14897
|
+
/** The actors whose commitments are FIRST-PERSON, and therefore the only ones minted. */
|
|
14898
|
+
const FIRST_PERSON_ACTORS = /* @__PURE__ */ new Set(["user", "agent"]);
|
|
14899
|
+
/**
|
|
14900
|
+
* A commitment this phase will act on, or the reason it was refused.
|
|
14901
|
+
*
|
|
14902
|
+
* Deterministic and between the model and the tree, the same position {@link refusalFor} occupies for a
|
|
14903
|
+
* candidate memory, and every clause is a real failure mode rather than a restatement of the schema:
|
|
14904
|
+
*
|
|
14905
|
+
* - **An actor outside `user`/`agent`.** Issue #44 asks for first-person commitments only, and the
|
|
14906
|
+
* contract's third value exists so a model has somewhere honest to put a third party's commitment
|
|
14907
|
+
* instead of mislabelling it. Dropping `other` HERE rather than refusing it in the schema is what
|
|
14908
|
+
* makes that honesty free: the model can report "a colleague said they'd ship it" accurately, and the
|
|
14909
|
+
* phase declines to open a task nobody in this store owes.
|
|
14910
|
+
* - **An empty statement.** It becomes the task's `<mark>` claim and therefore `files.gist`, so a
|
|
14911
|
+
* whitespace claim is a file the parser accepts and no search can find.
|
|
14912
|
+
* - **An empty quote or session id.** The quote is the reviewer's receipt in the commit message, and
|
|
14913
|
+
* the session is the task's `from_session` provenance. Neither is optional in the contract; this is
|
|
14914
|
+
* the redundancy every model-facing gate in this package carries, so a scripted or future
|
|
14915
|
+
* consolidator that skipped the schema still does not get past here.
|
|
14916
|
+
* - **Below the floor.** Counted separately by the caller rather than folded into the refusals,
|
|
14917
|
+
* because a night pressing against the floor is a different signal from a night sending malformed
|
|
14918
|
+
* commitments — the first says the threshold may be wrong and the second says the agent is.
|
|
14919
|
+
*
|
|
14920
|
+
* A session id OUTSIDE the batch is not checked here and is checked by the caller, which holds the
|
|
14921
|
+
* batch. See {@link commitmentSession}.
|
|
14922
|
+
*/
|
|
14923
|
+
const commitmentRefusalFor = (commitment) => {
|
|
14924
|
+
if (!FIRST_PERSON_ACTORS.has(commitment.actor)) return `actor ${commitment.actor} is not first-person`;
|
|
14925
|
+
if (commitment.statement.trim() === "") return "empty statement";
|
|
14926
|
+
if (commitment.evidence.quote.trim() === "") return "empty evidence quote";
|
|
14927
|
+
if (commitment.evidence.sessionId.trim() === "") return "empty evidence session";
|
|
14928
|
+
return null;
|
|
14929
|
+
};
|
|
14930
|
+
/**
|
|
14931
|
+
* A commitment's stable key: a normalized digest of the STATEMENT, and deliberately NOT of the session.
|
|
14932
|
+
*
|
|
14933
|
+
* This is the one place surface 2's keying departs from `task-detection`'s, which puts the source path
|
|
14934
|
+
* in its key, and the difference is forced by what closure has to do. The issue's requirement is that
|
|
14935
|
+
* "a commitment whose completion appears in A LATER SESSION can propose `task status done`" — so the
|
|
14936
|
+
* task a Monday session opened has to be findable from a Friday session's completion, and any key
|
|
14937
|
+
* carrying the session id makes those two keys different by construction. A session-keyed design cannot
|
|
14938
|
+
* close anything across nights, which is the only span closure is for.
|
|
14939
|
+
*
|
|
14940
|
+
* The consequence is that one sentence said in two sessions is ONE task, refreshed rather than
|
|
14941
|
+
* duplicated. That is the right reading for a commitment and the wrong one for `task-detection`'s
|
|
14942
|
+
* findings, and the asymmetry is not an inconsistency. A commitment is a piece of WORK: "wire the
|
|
14943
|
+
* capture path" promised on Monday and again on Wednesday is one thing to do, and two rows in the queue
|
|
14944
|
+
* would be one task and one duplicate. `task-detection`'s findings are per-MEMORY review decisions —
|
|
14945
|
+
* a corrected memory and its correction share most of their prose — and those are two files a reviewer
|
|
14946
|
+
* looks at separately, which is why the path belongs in that key.
|
|
14947
|
+
*
|
|
14948
|
+
* `detectionKey` normalizes (NFC, lowercase, collapsed whitespace), so a restatement whose spacing or
|
|
14949
|
+
* casing differs keys the same. It does not survive the model REWORDING the statement, which is the
|
|
14950
|
+
* honest limit of a digest over prose: `mintDetectedTask`'s frame-key check is the second net, the
|
|
14951
|
+
* volume cap is the third, and a completion whose wording moved is what `completionsUnmatched` counts.
|
|
14952
|
+
*/
|
|
14953
|
+
const commitmentKey = (commitment) => detectionKey(COMMITMENT_DETECTOR, commitment.statement);
|
|
14954
|
+
/**
|
|
14955
|
+
* The claim a commitment becomes: the work, stated as work, with the actor who owes it.
|
|
14956
|
+
*
|
|
14957
|
+
* **The STATEMENT leads, and that is a correctness requirement rather than a style choice.** The claim
|
|
14958
|
+
* is what `mintDetectedTask`'s frame-key proximity check reads, and the earlier wording — `confirm: the
|
|
14959
|
+
* <actor> committed to <statement>` — puts the statement in the rule's VALUE position: measured against
|
|
14960
|
+
* `frameKeyOf`, every commitment whose statement is six tokens or fewer keys on
|
|
14961
|
+
* `confirm: the agent committed to`, so "add the guard" and "ship the fix" shared a frame and the second
|
|
14962
|
+
* one answered `framed` and vanished. Only long statements escaped, by overflowing `MAX_VALUE_TOKENS` to
|
|
14963
|
+
* `null`, which made the collapse depend on statement length.
|
|
14964
|
+
*
|
|
14965
|
+
* With the statement in the frame the key carries it (measured: twelve statements across both actors,
|
|
14966
|
+
* twelve distinct keys, none null), so the check still fires between two DIFFERENT detectors describing
|
|
14967
|
+
* one commitment and never between two commitments of this one — which the statement digest in
|
|
14968
|
+
* {@link commitmentKey} already separates.
|
|
14969
|
+
*/
|
|
14970
|
+
const commitmentClaim = (commitment) => `confirm: ${flattenOne(commitment.statement)} is a commitment the ${commitment.actor} recorded and nothing says it is done.`;
|
|
14971
|
+
/** The title. The statement itself, which is already one sentence; `mintDetectedTask` cuts it to 90. */
|
|
14972
|
+
const commitmentTitle = (commitment) => `Commitment: ${flattenOne(commitment.statement)}`;
|
|
14973
|
+
/** Whitespace collapsed and one trailing sentence period dropped, so the claim reads as one clause. */
|
|
14974
|
+
const flattenOne = (text) => text.replace(/\s+/g, " ").trim().replace(/[.!?]+$/, "");
|
|
14975
|
+
/** The session a commitment cites, trimmed. The value the batch check and the key both read. */
|
|
14976
|
+
const commitmentSession = (commitment) => commitment.evidence.sessionId.trim();
|
|
14977
|
+
/**
|
|
14978
|
+
* The commit body for a batch of commitment tasks: one `commitment <session>: <quote>` line each.
|
|
14979
|
+
*
|
|
14980
|
+
* This is where a commitment's verbatim quote is allowed to go and nowhere else, the same rule
|
|
14981
|
+
* {@link commitContextFor} states for a candidate's evidence. A reviewer deciding whether a proposed
|
|
14982
|
+
* task is real needs the line it was read from, and a commit message is not part of the corpus: not
|
|
14983
|
+
* indexed, not chunked, not embedded, not retrievable. `commitPhase` indents the body, which is the
|
|
14984
|
+
* trailer-injection guard, and it matters here for the same reason it matters there — the text is a
|
|
14985
|
+
* model's, read out of a transcript nobody wrote for this system.
|
|
14986
|
+
*/
|
|
14987
|
+
const commitmentContext = (minted, closed) => [...minted.slice(0, COMMIT_EVIDENCE_LIMIT).map((one) => `commitment ${commitmentSession(one)}: ${one.evidence.quote.replace(/\s+/g, " ").slice(0, COMMIT_QUOTE_CHARS)}`), ...closed.map((path) => `closed ${path}: completion detected`)].join("\n");
|
|
14988
|
+
/** Every count at zero, so a phase that ran no commitment pass still reports the shape. */
|
|
14989
|
+
const ZERO_COMMITMENTS = {
|
|
14990
|
+
commitments: 0,
|
|
14991
|
+
commitmentTasks: 0,
|
|
14992
|
+
completionsApplied: 0,
|
|
14993
|
+
completionsUnmatched: 0,
|
|
14994
|
+
commitmentsSkipped: 0,
|
|
14995
|
+
commitmentsBelowFloor: 0,
|
|
14996
|
+
commitmentsRefreshed: 0,
|
|
14997
|
+
commitmentsFramed: 0,
|
|
14998
|
+
commitmentsDismissed: 0,
|
|
14999
|
+
commitmentsCapped: 0,
|
|
15000
|
+
staged: false,
|
|
15001
|
+
mintedCommitments: [],
|
|
15002
|
+
closedPaths: []
|
|
15003
|
+
};
|
|
15004
|
+
/**
|
|
15005
|
+
* The whole commitment pass: filter, then close what resolved and mint what did not.
|
|
15006
|
+
*
|
|
15007
|
+
* **Closures run BEFORE mints, and the order is load-bearing.** A resolved commitment and an unresolved
|
|
15008
|
+
* one can key the same when a model reports both readings of one sentence, and closing first means the
|
|
15009
|
+
* task leaves the open queue before the mint arm looks at it — so the mint opens a fresh task for a
|
|
15010
|
+
* commitment the same answer says is done, which reads as churn. Running mints first would instead
|
|
15011
|
+
* REFRESH the task and then immediately close it, which is worse: the queue loses a task in the same
|
|
15012
|
+
* commit that touched it, and the refresh's `memhtml-updated` stamp says a human was shown something
|
|
15013
|
+
* that was archived before they could look. Ordering closures first makes a same-answer contradiction
|
|
15014
|
+
* resolve to "closed", which is the reading that costs a reviewer nothing.
|
|
15015
|
+
*
|
|
15016
|
+
* **Only sessions in the BATCH.** The client already refuses a turn citing a session it did not make
|
|
15017
|
+
* readable (`ungroundedCommitmentReason`), and this narrows the same way `analyzedFrom` narrows the
|
|
15018
|
+
* watermark set: an id outside the batch this phase asked about is a bug in the consolidator, and it
|
|
15019
|
+
* must not become a task file whose provenance names a session nobody selected. Cheap, so unconditional.
|
|
15020
|
+
*
|
|
15021
|
+
* **The budget is the run's shared one**, taken once here and threaded, per `budgetFor`'s contract.
|
|
15022
|
+
* Overflow lands in `budget.overflow`, which the caller reports as `capped` alongside every other
|
|
15023
|
+
* detector's.
|
|
15024
|
+
*/
|
|
15025
|
+
const consolidateCommitments = (env, commitments, batchSessionIds) => Effect.gen(function* () {
|
|
15026
|
+
if (commitments.length === 0) return ZERO_COMMITMENTS;
|
|
15027
|
+
let skipped = 0;
|
|
15028
|
+
let belowFloor = 0;
|
|
15029
|
+
/** Resolved commitments the floor turned away: completions this night declined to apply. */
|
|
15030
|
+
let belowFloorCompletions = 0;
|
|
15031
|
+
const admissible = [];
|
|
15032
|
+
for (const [offset, commitment] of commitments.entries()) {
|
|
15033
|
+
const refusal = commitmentRefusalFor(commitment);
|
|
15034
|
+
if (refusal !== null) {
|
|
15035
|
+
yield* Effect.logWarning(`sleep.trace-consolidation commitment ${offset} skipped: ${refusal}`);
|
|
15036
|
+
skipped += 1;
|
|
15037
|
+
continue;
|
|
15038
|
+
}
|
|
15039
|
+
if (!batchSessionIds.has(commitmentSession(commitment))) {
|
|
15040
|
+
yield* Effect.logWarning(`sleep.trace-consolidation commitment ${offset} skipped: session ${commitmentSession(commitment)} is not in this run's batch`);
|
|
15041
|
+
skipped += 1;
|
|
15042
|
+
continue;
|
|
15043
|
+
}
|
|
15044
|
+
if (commitment.confidence < .7) {
|
|
15045
|
+
belowFloor += 1;
|
|
15046
|
+
/**
|
|
15047
|
+
* A resolved commitment below the floor is the issue's "left for review" case, so it is counted
|
|
15048
|
+
* as an unapplied completion HERE rather than inferred later by subtraction.
|
|
15049
|
+
*
|
|
15050
|
+
* Only the ones that reached the floor. A commitment the filter refused above — a third party's,
|
|
15051
|
+
* or one naming a session outside the batch — is not a completion this store declined to apply;
|
|
15052
|
+
* it was never a first-person commitment at all, and counting it as an unmatched completion would
|
|
15053
|
+
* report the same finding under two counters and make `completionsUnmatched` read as a keying
|
|
15054
|
+
* problem on a night whose only fault was a mislabelled actor.
|
|
15055
|
+
*/
|
|
15056
|
+
if (commitment.resolved) belowFloorCompletions += 1;
|
|
15057
|
+
continue;
|
|
15058
|
+
}
|
|
15059
|
+
admissible.push(commitment);
|
|
15060
|
+
}
|
|
15061
|
+
/**
|
|
15062
|
+
* The closure arm. The open queue is read ONCE for the whole batch and then narrowed in memory:
|
|
15063
|
+
* `openDetections` is a `readdir` plus a parse per file, and asking it per resolved commitment
|
|
15064
|
+
* would be the round-trip-per-row shape every batch read in this package exists to avoid.
|
|
15065
|
+
*/
|
|
15066
|
+
const resolved = admissible.filter((commitment) => commitment.resolved);
|
|
15067
|
+
const closedPaths = [];
|
|
15068
|
+
let unmatched = 0;
|
|
15069
|
+
if (resolved.length > 0) {
|
|
15070
|
+
const open = yield* openDetections(env);
|
|
15071
|
+
const byKey = new Map(open.map((detected) => [detected.key, detected]));
|
|
15072
|
+
for (const commitment of resolved) {
|
|
15073
|
+
const match = byKey.get(commitmentKey(commitment));
|
|
15074
|
+
if (match === void 0) {
|
|
15075
|
+
unmatched += 1;
|
|
15076
|
+
continue;
|
|
15077
|
+
}
|
|
15078
|
+
/**
|
|
15079
|
+
* `closeDetectedTask` re-checks the path, which is redundant with `openDetections` only
|
|
15080
|
+
* returning detected paths and is kept for the reason that function's own note gives: the guard
|
|
15081
|
+
* belongs at the write, not at the lookup. A `false` here means the file vanished between the
|
|
15082
|
+
* read and the write, so it is counted as unmatched rather than as a closure.
|
|
15083
|
+
*/
|
|
15084
|
+
if (yield* closeDetectedTask(env, match.path)) closedPaths.push(match.path);
|
|
15085
|
+
else unmatched += 1;
|
|
15086
|
+
byKey.delete(match.key);
|
|
15087
|
+
}
|
|
15088
|
+
}
|
|
15089
|
+
/**
|
|
15090
|
+
* The completions the floor turned away, added to the ones that matched nothing.
|
|
15091
|
+
*
|
|
15092
|
+
* ADDED rather than derived by subtracting `resolved.length` from the resolved commitments in the
|
|
15093
|
+
* whole answer, which is what an earlier version did and got wrong: that difference also swept in
|
|
15094
|
+
* every resolved commitment the FILTER refused, so a night whose only fault was a third party's
|
|
15095
|
+
* completion reported an unmatched completion and pointed an operator at the keying.
|
|
15096
|
+
*/
|
|
15097
|
+
unmatched += belowFloorCompletions;
|
|
15098
|
+
const budget = budgetFor(env);
|
|
15099
|
+
/** The shared counter BEFORE this pass, so `commitmentsCapped` is this pass's own delta. */
|
|
15100
|
+
const overflowBefore = budget.overflow;
|
|
15101
|
+
const minted = [];
|
|
15102
|
+
let refreshed = 0;
|
|
15103
|
+
let framed = 0;
|
|
15104
|
+
let dismissed = 0;
|
|
15105
|
+
for (const commitment of admissible) {
|
|
15106
|
+
if (commitment.resolved) continue;
|
|
15107
|
+
const outcome = yield* mintDetectedTask(env, budget, {
|
|
15108
|
+
detector: COMMITMENT_DETECTOR,
|
|
15109
|
+
/**
|
|
15110
|
+
* The statement alone, matching {@link commitmentKey} exactly. `mintDetectedTask` re-derives the
|
|
15111
|
+
* digest from `detector` + `finding`, so a `finding` that disagreed with the key this phase
|
|
15112
|
+
* matches closures against would mint under one path and look for another — the arms would
|
|
15113
|
+
* silently never meet. One expression rather than two is what keeps them the same key.
|
|
15114
|
+
*/
|
|
15115
|
+
finding: commitment.statement,
|
|
15116
|
+
title: commitmentTitle(commitment),
|
|
15117
|
+
claim: commitmentClaim(commitment),
|
|
15118
|
+
detail: `Recorded in a consolidated session at confidence ${commitment.confidence.toFixed(2)} and never stated as done. Confirm it is still wanted, or close it.`,
|
|
15119
|
+
evidence: {
|
|
15120
|
+
kind: "session",
|
|
15121
|
+
sessionId: commitmentSession(commitment),
|
|
15122
|
+
statement: commitment.statement
|
|
15123
|
+
},
|
|
15124
|
+
...typeof commitment.dueHint === "string" ? { dueHint: commitment.dueHint } : {}
|
|
15125
|
+
});
|
|
15126
|
+
if (outcome === "minted") minted.push(commitment);
|
|
15127
|
+
else if (outcome === "refreshed") refreshed += 1;
|
|
15128
|
+
else if (outcome === "framed") framed += 1;
|
|
15129
|
+
else if (outcome === "dismissed") dismissed += 1;
|
|
15130
|
+
}
|
|
15131
|
+
return {
|
|
15132
|
+
commitments: commitments.length,
|
|
15133
|
+
commitmentTasks: minted.length,
|
|
15134
|
+
completionsApplied: closedPaths.length,
|
|
15135
|
+
completionsUnmatched: unmatched,
|
|
15136
|
+
commitmentsSkipped: skipped,
|
|
15137
|
+
commitmentsBelowFloor: belowFloor,
|
|
15138
|
+
commitmentsRefreshed: refreshed,
|
|
15139
|
+
commitmentsFramed: framed,
|
|
15140
|
+
commitmentsDismissed: dismissed,
|
|
15141
|
+
commitmentsCapped: budget.overflow - overflowBefore,
|
|
15142
|
+
staged: minted.length > 0 || refreshed > 0 || closedPaths.length > 0,
|
|
15143
|
+
mintedCommitments: minted,
|
|
15144
|
+
closedPaths
|
|
15145
|
+
};
|
|
15146
|
+
});
|
|
15147
|
+
/**
|
|
13079
15148
|
* A candidate the phase will write, or `null` with the reason it was refused.
|
|
13080
15149
|
*
|
|
13081
15150
|
* The gate is deterministic and sits between the agent and the tree, which is where every
|
|
@@ -13178,10 +15247,7 @@ const traceConsolidation = (env) => Effect.gen(function* () {
|
|
|
13178
15247
|
if (consolidator === void 0) return {
|
|
13179
15248
|
...emptyOutcome({
|
|
13180
15249
|
...base,
|
|
13181
|
-
|
|
13182
|
-
candidates: 0,
|
|
13183
|
-
written: 0,
|
|
13184
|
-
consolidated: 0
|
|
15250
|
+
...ZERO_COUNTS
|
|
13185
15251
|
}),
|
|
13186
15252
|
detail: "no consolidator bound"
|
|
13187
15253
|
};
|
|
@@ -13204,10 +15270,7 @@ const traceConsolidation = (env) => Effect.gen(function* () {
|
|
|
13204
15270
|
});
|
|
13205
15271
|
if (batch.length === 0) return emptyOutcome({
|
|
13206
15272
|
...base,
|
|
13207
|
-
|
|
13208
|
-
candidates: 0,
|
|
13209
|
-
written: 0,
|
|
13210
|
-
consolidated: 0
|
|
15273
|
+
...ZERO_COUNTS
|
|
13211
15274
|
});
|
|
13212
15275
|
/**
|
|
13213
15276
|
* A dry run stops HERE, having done the whole deterministic half: the batch is real and counted,
|
|
@@ -13217,10 +15280,8 @@ const traceConsolidation = (env) => Effect.gen(function* () {
|
|
|
13217
15280
|
*/
|
|
13218
15281
|
if (env.dryRun) return emptyOutcome({
|
|
13219
15282
|
...base,
|
|
13220
|
-
|
|
13221
|
-
|
|
13222
|
-
written: 0,
|
|
13223
|
-
consolidated: 0
|
|
15283
|
+
...ZERO_COUNTS,
|
|
15284
|
+
batch: batch.length
|
|
13224
15285
|
});
|
|
13225
15286
|
/**
|
|
13226
15287
|
* The whole consolidator call in isolation. A failure is a VALUE here and not a phase
|
|
@@ -13253,10 +15314,8 @@ const traceConsolidation = (env) => Effect.gen(function* () {
|
|
|
13253
15314
|
return {
|
|
13254
15315
|
...emptyOutcome({
|
|
13255
15316
|
...base,
|
|
13256
|
-
|
|
13257
|
-
|
|
13258
|
-
written: 0,
|
|
13259
|
-
consolidated: 0
|
|
15317
|
+
...ZERO_COUNTS,
|
|
15318
|
+
batch: batch.length
|
|
13260
15319
|
}),
|
|
13261
15320
|
detail: `consolidator unavailable: ${failure._tag}`
|
|
13262
15321
|
};
|
|
@@ -13337,6 +15396,29 @@ const traceConsolidation = (env) => Effect.gen(function* () {
|
|
|
13337
15396
|
written += 1;
|
|
13338
15397
|
}
|
|
13339
15398
|
/**
|
|
15399
|
+
* Surface 2, AFTER every candidate commit and BEFORE the watermark.
|
|
15400
|
+
*
|
|
15401
|
+
* After the candidates, so a commitment task cannot ride into a `distill …` commit and confuse what
|
|
15402
|
+
* that commit decided; each half of the answer gets its own reviewable commit. Before the watermark,
|
|
15403
|
+
* for the reason the watermark's own note gives: it goes last, so a process killed mid-phase
|
|
15404
|
+
* re-reads the batch rather than recording it read with nothing to show.
|
|
15405
|
+
*
|
|
15406
|
+
* The batch is the grounding set, `analyzedFrom` is not. A commitment cites a session whose
|
|
15407
|
+
* TRANSCRIPT was read, and `analyzedSessionIds` is the reachable set the CLIENT computed — which is
|
|
15408
|
+
* the right input for a watermark and the wrong one for this check, since a scripted or degraded
|
|
15409
|
+
* consolidator could report a narrower reachable set while still having read the sessions it quotes.
|
|
15410
|
+
* The batch is what this phase asked about, and it is the containment the phase can assert.
|
|
15411
|
+
*/
|
|
15412
|
+
const commitments = yield* consolidateCommitments(env, outcome.success.commitments, new Set(batch.map((session) => session.session_id)));
|
|
15413
|
+
if (commitments.staged) {
|
|
15414
|
+
const commitSha = yield* commitPhase(env, "trace-consolidation", `detect ${String(commitments.commitmentTasks)} commitments, close ${String(commitments.completionsApplied)} completed`, {
|
|
15415
|
+
...base,
|
|
15416
|
+
batch: batch.length,
|
|
15417
|
+
...commitmentCounts(commitments)
|
|
15418
|
+
}, commitmentContext(commitments.mintedCommitments, commitments.closedPaths));
|
|
15419
|
+
if (commitSha !== null) lastCommit = commitSha;
|
|
15420
|
+
}
|
|
15421
|
+
/**
|
|
13340
15422
|
* The watermark is written LAST, after every commit, and covers exactly the sessions the agent
|
|
13341
15423
|
* ACTUALLY READ. {@link analyzedFrom} is that set, and it is not `batch`.
|
|
13342
15424
|
*
|
|
@@ -13394,13 +15476,52 @@ const traceConsolidation = (env) => Effect.gen(function* () {
|
|
|
13394
15476
|
skipped,
|
|
13395
15477
|
conflicts: conflicted,
|
|
13396
15478
|
consolidated: analyzed.length,
|
|
13397
|
-
unreachable
|
|
15479
|
+
unreachable,
|
|
15480
|
+
...commitmentCounts(commitments)
|
|
13398
15481
|
},
|
|
13399
15482
|
commitSha: lastCommit,
|
|
13400
15483
|
llmCalls
|
|
13401
15484
|
};
|
|
13402
15485
|
});
|
|
13403
15486
|
/**
|
|
15487
|
+
* The commitment half of the counts, from the pass's outcome.
|
|
15488
|
+
*
|
|
15489
|
+
* One function, called by both the phase's return and the commitment commit's trailer, so a reader
|
|
15490
|
+
* comparing the `Memhtml-Counts` trailer against the report sees the same keys with the same meanings.
|
|
15491
|
+
* `capped` is the SHARED budget's overflow — every detector's, not this one's, per `DETECTED_TASK_CAP`'s
|
|
15492
|
+
* note — so it is read off the budget rather than counted here.
|
|
15493
|
+
*/
|
|
15494
|
+
const commitmentCounts = (outcome) => ({
|
|
15495
|
+
commitments: outcome.commitments,
|
|
15496
|
+
commitmentTasks: outcome.commitmentTasks,
|
|
15497
|
+
completionsApplied: outcome.completionsApplied,
|
|
15498
|
+
completionsUnmatched: outcome.completionsUnmatched,
|
|
15499
|
+
commitmentsSkipped: outcome.commitmentsSkipped,
|
|
15500
|
+
commitmentsBelowFloor: outcome.commitmentsBelowFloor,
|
|
15501
|
+
commitmentsRefreshed: outcome.commitmentsRefreshed,
|
|
15502
|
+
commitmentsFramed: outcome.commitmentsFramed,
|
|
15503
|
+
commitmentsDismissed: outcome.commitmentsDismissed,
|
|
15504
|
+
commitmentsCapped: outcome.commitmentsCapped
|
|
15505
|
+
});
|
|
15506
|
+
/**
|
|
15507
|
+
* The full count SHAPE, at zero, for every path that returns before the model answer.
|
|
15508
|
+
*
|
|
15509
|
+
* Every key the phase can report is present on every path, because a report reader comparing two nights
|
|
15510
|
+
* reads a missing key as a phase that does not have the concept rather than as a night that did none of
|
|
15511
|
+
* it. Same rule `task-detection`'s `ZERO` and `edge-typing`'s `zero` state. `base` is spread beside it
|
|
15512
|
+
* because those three counters are real on every path, including a dry run.
|
|
15513
|
+
*/
|
|
15514
|
+
const ZERO_COUNTS = {
|
|
15515
|
+
batch: 0,
|
|
15516
|
+
candidates: 0,
|
|
15517
|
+
written: 0,
|
|
15518
|
+
skipped: 0,
|
|
15519
|
+
conflicts: 0,
|
|
15520
|
+
consolidated: 0,
|
|
15521
|
+
unreachable: 0,
|
|
15522
|
+
...commitmentCounts(ZERO_COMMITMENTS)
|
|
15523
|
+
};
|
|
15524
|
+
/**
|
|
13404
15525
|
* The sessions to watermark: those the agent reported analyzing, INTERSECTED with the batch.
|
|
13405
15526
|
*
|
|
13406
15527
|
* The intersection is the containment half of the invariant and it is cheap, so it is unconditional. A
|
|
@@ -13563,6 +15684,7 @@ const PHASE_BODIES = {
|
|
|
13563
15684
|
compress,
|
|
13564
15685
|
reprieve,
|
|
13565
15686
|
"trace-consolidation": traceConsolidation,
|
|
15687
|
+
"task-detection": taskDetection,
|
|
13566
15688
|
integrity,
|
|
13567
15689
|
"state-export": stateExport,
|
|
13568
15690
|
report: reportPhase([])
|
|
@@ -13613,7 +15735,17 @@ const run = (deps, options) => Effect.gen(function* () {
|
|
|
13613
15735
|
date: options.date,
|
|
13614
15736
|
at: instant.at,
|
|
13615
15737
|
atMillis: instant.millis,
|
|
13616
|
-
dryRun
|
|
15738
|
+
dryRun,
|
|
15739
|
+
/**
|
|
15740
|
+
* ONE budget for the whole run, created here and shared by every phase that mints a detected
|
|
15741
|
+
* task. `DETECTED_TASK_CAP` bounds the NIGHT and not each detector, because how many proposals a
|
|
15742
|
+
* human can review is a property of the human — so a night where entity resolution finds nine
|
|
15743
|
+
* review candidates leaves task detection one, first come.
|
|
15744
|
+
*
|
|
15745
|
+
* Created per run rather than held in a module, which is what keeps two runs in one process (and
|
|
15746
|
+
* two tests in one file) from sharing a counter.
|
|
15747
|
+
*/
|
|
15748
|
+
detectionBudget: makeDetectionBudget()
|
|
13617
15749
|
};
|
|
13618
15750
|
if (!dryRun) yield* deps.git.checkoutBranch(runId, { create: true }).pipe(Effect.orElseSucceed(() => {}));
|
|
13619
15751
|
yield* ignoreFailure(recordRun(deps.db, {
|
|
@@ -13671,7 +15803,16 @@ const resume = (deps, runId, options = {}) => Effect.gen(function* () {
|
|
|
13671
15803
|
date,
|
|
13672
15804
|
at: instant.at,
|
|
13673
15805
|
atMillis: instant.millis,
|
|
13674
|
-
dryRun: false
|
|
15806
|
+
dryRun: false,
|
|
15807
|
+
/**
|
|
15808
|
+
* A resume gets a FRESH budget, deliberately. The alternative would be reconstructing how much
|
|
15809
|
+
* the interrupted attempt spent by counting detected tasks in the tree, and the count would be
|
|
15810
|
+
* wrong in the direction that matters: a phase that minted three and was then killed would have
|
|
15811
|
+
* its own three counted against it on the retry, so a resume would mint fewer than the run it is
|
|
15812
|
+
* finishing. The cost of a fresh one is bounded by the cap, and the mints a resume repeats are
|
|
15813
|
+
* refreshes rather than duplicates, which cost no budget at all.
|
|
15814
|
+
*/
|
|
15815
|
+
detectionBudget: makeDetectionBudget()
|
|
13675
15816
|
};
|
|
13676
15817
|
const remaining = SLEEP_PHASES.filter((phase) => !completed.has(phase));
|
|
13677
15818
|
const executed = yield* executePhases(env, remaining, /* @__PURE__ */ new Set());
|
|
@@ -13679,7 +15820,7 @@ const resume = (deps, runId, options = {}) => Effect.gen(function* () {
|
|
|
13679
15820
|
const ended = yield* nowIso;
|
|
13680
15821
|
/**
|
|
13681
15822
|
* Skipped-because-already-done rows are reported explicitly, so a resume's report accounts for all
|
|
13682
|
-
*
|
|
15823
|
+
* sixteen phases. A report that showed only the eight it ran would read as a partial run.
|
|
13683
15824
|
*/
|
|
13684
15825
|
const priorRows = yield* ignoreFailureWith(readPhases(deps.db, runId), []);
|
|
13685
15826
|
const already = [...completed].map((phase) => {
|
|
@@ -14719,5 +16860,5 @@ const earliest = (left, right) => left === null ? right : right === null ? left
|
|
|
14719
16860
|
const latest = (left, right) => left === null ? right : right === null ? left : left >= right ? left : right;
|
|
14720
16861
|
|
|
14721
16862
|
//#endregion
|
|
14722
|
-
export {
|
|
14723
|
-
//# sourceMappingURL=dist-
|
|
16863
|
+
export { STATE_DB_PATH as $, IndexRecorder as A, IndexGit as B, ModelClient as C, EmbeddingsLive as D, Embeddings as E, makeRetrieval as F, MIGRATIONS_DIR as G, sanitizeFtsQuery as H, reinforce as I, Store as J, STATE_MIGRATIONS_DIR as K, Indexer as L, persistScanned as M, readWatermark as N, EMBED_DIM as O, Retrieval as P, SLEEP_REPORTS_DIR as Q, makeIndexer as R, runDiscrimination as S, wrapAsData as T, DatabaseService as U, makeGitPort as V, makeDatabase as W, makeStore as X, expandRoot as Y, INDEX_DB_PATH as Z, meta as _, parseSidecar as a, makeGit as at, isSleepPhase as b, generateArtifacts as c, setMeta as ct, allPaths as d, fenceOpeningOf as dt, STATE_SIDECAR_PATH as et, danglingEdges as f, REINFORCE_SIGNALS as ft, link as g, hrefFor as h, makeSleep as i, Git as it, makeIndexRecorder as j, EMBED_WATERMARK as k, DETECTION_PREFIX as l, isValidDatetime as lt, applyHeadEdits as m, scanTraceRoot as n, initRepo as nt, renderSidecar as o, commitSubject as ot, publishRows as p, frameKeyOf as pt, STATE_SCHEMA as q, Sleep as r, readFileOrNull as rt, archivedFormOf as s, checkMemory as st, mergeTailExtract as t, attemptIo as tt, accessRows as u, closesFence as ut, unlink as v, ModelClientLive as w, discriminationGate as x, SLEEP_PHASES as y, readIndexState as z };
|
|
16864
|
+
//# sourceMappingURL=dist-D5DlgqH2.mjs.map
|