@dsh-enhanced/assistant-evaluation 0.1.65 → 0.1.66
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +54 -1
- package/lib/attention-owner-questions.d.ts +85 -0
- package/lib/attention-owner-questions.d.ts.map +1 -0
- package/lib/attention-owner-questions.js +142 -0
- package/lib/attention-owner-questions.js.map +1 -0
- package/lib/attention-tuning.d.ts +431 -0
- package/lib/attention-tuning.d.ts.map +1 -0
- package/lib/attention-tuning.js +892 -0
- package/lib/attention-tuning.js.map +1 -0
- package/lib/benchmark/cli.d.ts.map +1 -1
- package/lib/benchmark/cli.js +11 -6
- package/lib/benchmark/cli.js.map +1 -1
- package/lib/benchmark/corpus.d.ts +13 -2
- package/lib/benchmark/corpus.d.ts.map +1 -1
- package/lib/benchmark/corpus.js +19 -9
- package/lib/benchmark/corpus.js.map +1 -1
- package/lib/benchmark/families-corpus.d.ts +11 -0
- package/lib/benchmark/families-corpus.d.ts.map +1 -0
- package/lib/benchmark/families-corpus.js +133 -0
- package/lib/benchmark/families-corpus.js.map +1 -0
- package/lib/benchmark/holdout-isolation.d.ts +27 -0
- package/lib/benchmark/holdout-isolation.d.ts.map +1 -0
- package/lib/benchmark/holdout-isolation.js +66 -0
- package/lib/benchmark/holdout-isolation.js.map +1 -0
- package/lib/benchmark/holdout.d.ts.map +1 -1
- package/lib/benchmark/holdout.js +3 -0
- package/lib/benchmark/holdout.js.map +1 -1
- package/lib/benchmark/index.d.ts +3 -0
- package/lib/benchmark/index.d.ts.map +1 -1
- package/lib/benchmark/index.js +2 -0
- package/lib/benchmark/index.js.map +1 -1
- package/lib/benchmark/native-skills.d.ts +12 -14
- package/lib/benchmark/native-skills.d.ts.map +1 -1
- package/lib/benchmark/native.d.ts +15 -2
- package/lib/benchmark/native.d.ts.map +1 -1
- package/lib/benchmark/native.js +67 -14
- package/lib/benchmark/native.js.map +1 -1
- package/lib/benchmark/report.d.ts.map +1 -1
- package/lib/benchmark/report.js +98 -8
- package/lib/benchmark/report.js.map +1 -1
- package/lib/benchmark/runner.d.ts.map +1 -1
- package/lib/benchmark/runner.js +5 -0
- package/lib/benchmark/runner.js.map +1 -1
- package/lib/benchmark/types.d.ts +64 -7
- package/lib/benchmark/types.d.ts.map +1 -1
- package/lib/index.d.ts +7 -0
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +4 -0
- package/lib/index.js.map +1 -1
- package/lib/intake-attention-agreement.d.ts +63 -0
- package/lib/intake-attention-agreement.d.ts.map +1 -0
- package/lib/intake-attention-agreement.js +146 -0
- package/lib/intake-attention-agreement.js.map +1 -0
- package/lib/notification-outcomes.d.ts +56 -0
- package/lib/notification-outcomes.d.ts.map +1 -0
- package/lib/notification-outcomes.js +348 -0
- package/lib/notification-outcomes.js.map +1 -0
- package/lib/service.d.ts +244 -2
- package/lib/service.d.ts.map +1 -1
- package/lib/service.js +830 -5
- package/lib/service.js.map +1 -1
- package/lib/sqlite.d.ts +8 -2
- package/lib/sqlite.d.ts.map +1 -1
- package/lib/sqlite.js +406 -17
- package/lib/sqlite.js.map +1 -1
- package/lib/store.d.ts +13 -2
- package/lib/store.d.ts.map +1 -1
- package/lib/store.js +120 -24
- package/lib/store.js.map +1 -1
- package/lib/types.d.ts +247 -5
- package/lib/types.d.ts.map +1 -1
- package/lib/types.js +11 -0
- package/lib/types.js.map +1 -1
- package/lib/version.d.ts +1 -1
- package/lib/version.js +1 -1
- package/lib/work-item-owner-proof.d.ts +4 -0
- package/lib/work-item-owner-proof.d.ts.map +1 -0
- package/lib/work-item-owner-proof.js +53 -0
- package/lib/work-item-owner-proof.js.map +1 -0
- package/package.json +30 -30
package/lib/store.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../src/store.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../src/store.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAA;AAC7D,OAAO,EAAE,4BAA4B,EAAE,MAAM,gCAAgC,CAAA;AAC7E,OAAO,EAAE,yBAAyB,EAAE,MAAM,4BAA4B,CAAA;AAWtE,OAAO,KAAK,EAGV,sCAAsC,EACtC,qBAAqB,EACrB,gBAAgB,EAEhB,6BAA6B,EAC7B,mCAAmC,EAEnC,+BAA+B,EAC/B,yBAAyB,EACzB,eAAe,EACf,eAAe,EACf,eAAe,EACf,qBAAqB,EACrB,mBAAmB,EACnB,eAAe,EACf,YAAY,EACZ,iBAAiB,EACjB,cAAc,EACd,mBAAmB,EAEnB,gBAAgB,EAChB,mBAAmB,EACnB,oBAAoB,EACpB,aAAa,EACb,oCAAoC,EACpC,qCAAqC,EACrC,4BAA4B,EAC7B,MAAM,YAAY,CAAA;AAEnB,MAAM,MAAM,wBAAwB,GAAG,sBAAsB,GAAG,eAAe,GAAG,WAAW,GAAG,kBAAkB,CAAA;AAElH,qBAAa,oBAAqB,SAAQ,KAAK;IACjC,QAAQ,CAAC,IAAI,EAAE,wBAAwB;gBAA9B,IAAI,EAAE,wBAAwB,EAAE,OAAO,EAAE,MAAM;CAIrE;AAED,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,CAAC,EAAE,MAAM,MAAM,CAAA;IAClB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,kBAAkB,CAAC,EAAE,MAAM,CAAA;IAC3B,sBAAsB,CAAC,EAAE,MAAM,CAAA;CAChC;AAgHD,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,eAAe,GAAG;IAAE,KAAK,EAAE,eAAe,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAO7G;AA0LD,4EAA4E;AAC5E,wBAAgB,kCAAkC,CAAC,KAAK,EAAE;IACxD,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,SAAS,CAAC,EAAE,QAAQ,CAAC;QACnB,SAAS,EAAE,MAAM,CAAA;QACjB,MAAM,EAAE,eAAe,CAAA;QACvB,MAAM,EAAE,QAAQ,CAAC;YAAE,IAAI,EAAE,iBAAiB,CAAC;YAAC,EAAE,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;QACzD,QAAQ,EAAE,SAAS,QAAQ,CAAC,qBAAqB,CAAC,EAAE,CAAA;QACpD,UAAU,EAAE,MAAM,CAAA;QAClB,SAAS,EAAE,QAAQ,CAAC;YAAE,EAAE,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KACrD,CAAC,CAAA;IACF,SAAS,CAAC,EAAE,QAAQ,CAAC;QACnB,SAAS,EAAE,MAAM,CAAA;QACjB,MAAM,EAAE,eAAe,CAAA;QACvB,MAAM,EAAE,QAAQ,CAAC;YAAE,IAAI,EAAE,iBAAiB,CAAC;YAAC,EAAE,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;QACzD,QAAQ,EAAE,SAAS,QAAQ,CAAC,qBAAqB,CAAC,EAAE,CAAA;QACpD,UAAU,EAAE,MAAM,CAAA;QAClB,SAAS,EAAE,QAAQ,CAAC;YAAE,EAAE,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KACrD,CAAC,CAAA;IACF,UAAU,EAAE,QAAQ,CAAC;QACnB,WAAW,EAAE,gBAAgB,GAAG,iBAAiB,GAAG,WAAW,GAAG,cAAc,GAAG,WAAW,GAAG,SAAS,CAAA;QAC1G,UAAU,EAAE,MAAM,CAAA;QAClB,WAAW,EAAE,QAAQ,GAAG,SAAS,CAAA;QACjC,iBAAiB,CAAC,EAAE,MAAM,CAAA;KAC3B,CAAC,CAAA;CACH,GAAG,MAAM,CA8BT;AAgJD,qBAAa,eAAe;;IAS1B,qDAAqD;IACrD,QAAQ,CAAC,aAAa,EAAE,yBAAyB,CAAA;IACjD,uFAAuF;IACvF,QAAQ,CAAC,SAAS,EAAE,qBAAqB,CAAA;IACzC,2FAA2F;IAC3F,QAAQ,CAAC,kBAAkB,EAAE,4BAA4B,CAAA;gBAE7C,OAAO,EAAE,sBAAsB;IAmC3C,KAAK,IAAI,IAAI;IAEb,UAAU,CAAC,UAAU,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS;IAS1F,sFAAsF;IACtF,wBAAwB,CAAC,MAAM,EAAE,QAAQ,CAAC,OAAO,YAAY,EAAE,+BAA+B,CAAC,GAAG,IAAI;IA4BtG;;;;OAIG;IACH,oCAAoC,CAClC,MAAM,EAAE,QAAQ,CAAC,OAAO,YAAY,EAAE,+BAA+B,CAAC,EACtE,KAAK,EAAE,QAAQ,CAAC,4BAA4B,CAAC,EAC7C,eAAe,EAAE,QAAQ,CAAC,eAAe,CAAC,GACzC,OAAO;IAsIV,mBAAmB,CAAC,KAAK,EAAE,eAAe,EAAE,KAAK,EAAE,MAAM,EAAE,iBAAiB,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,GAAG,mBAAmB,GAAG,SAAS;IAChJ,mBAAmB,CAAC,KAAK,EAAE,eAAe,EAAE,WAAW,EAAE,gBAAgB,GAAG,iBAAiB,GAAG,cAAc,GAAG,WAAW,EAAE,UAAU,EAAE,MAAM,EAAE,iBAAiB,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,GAAG,mBAAmB,GAAG,SAAS;IAoBvO,OAAO,CAAC,kBAAkB;IAM1B,MAAM,CAAC,KAAK,EAAE,eAAe,EAAE,YAAY,CAAC,EAAE,QAAQ,CAAC,qBAAqB,CAAC,GAAG,aAAa;IAgI7F,sBAAsB,CAAC,UAAU,SAAM,EAAE,QAAQ,SAAc,GAAG,+BAA+B,EAAE;IAiBnG,qBAAqB,CAAC,UAAU,EAAE,eAAe,EAAE,QAAQ,SAAc,GAAG,+BAA+B,GAAG,SAAS;IAevH,aAAa,CAAC,UAAU,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,GAAG,yBAAyB,GAAG,SAAS;IAa5G,kBAAkB,CAAC,KAAK,EAAE;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO;IAUzE,eAAe,CAAC,KAAK,EAAE;QACrB,YAAY,EAAE,MAAM,CAAA;QACpB,GAAG,EAAE,MAAM,CAAA;QACX,OAAO,EAAE,MAAM,CAAA;QACf,WAAW,EAAE,MAAM,CAAA;KACpB,GAAG,OAAO;IAiBX;;;;OAIG;IACH,oBAAoB,CAAC,KAAK,EAAE,mBAAmB,GAAG,oBAAoB;IA2DtE,6EAA6E;IAC7E,qBAAqB,CACnB,UAAU,EAAE,eAAe,EAC3B,eAAe,EAAE,SAAS,MAAM,EAAE,GACjC,oBAAoB,EAAE;IA6BzB,0EAA0E;IAC1E,iBAAiB,CAAC,UAAU,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,GAAG,gBAAgB,GAAG,SAAS;IAapG,oFAAoF;IACpF,kCAAkC,CAChC,UAAU,EAAE,eAAe,EAC3B,UAAU,EAAE,MAAM,GACjB,oCAAoC,GAAG,SAAS;IAcnD,4EAA4E;IAC5E,+BAA+B,CAAC,UAAU,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,GAAG,oCAAoC,GAAG,SAAS;IAmBpI,8FAA8F;IAC9F,6BAA6B,CAAC,UAAU,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,GAAG,oCAAoC,GAAG,SAAS;IAYjI,mEAAmE;IACnE,gCAAgC,CAC9B,UAAU,EAAE,eAAe,EAC3B,iBAAiB,EAAE,MAAM,GACxB,oCAAoC,GAAG,SAAS;IAanD;;;;OAIG;IACH,yBAAyB,CACvB,UAAU,EAAE,eAAe,EAC3B,cAAc,EAAE,MAAM,GACrB,oCAAoC,GAAG,SAAS;IAsDnD,8BAA8B,CAC5B,UAAU,EAAE,eAAe,EAC3B,UAAU,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,EAC/D,UAAU,SAAM,GACf,qCAAqC;IAuCxC,wBAAwB,CAAC,UAAU,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,EAAE,sBAAsB,EAAE,MAAM,EAAE,qBAAqB,EAAE,MAAM,GAAG,QAAQ,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,SAAS,GAAG,SAAS,GAAG,UAAU,CAAC;QAAC,eAAe,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,GAAG,SAAS;IA0BrS;;;;;OAKG;IACH,uBAAuB,CAAC,CAAC,EACvB,UAAU,EAAE,eAAe,EAC3B,UAAU,EAAE,6BAA6B,EACzC,QAAQ,EAAE,MAAM,CAAC,EACjB,OAAO,GAAE,QAAQ,CAAC;QAAE,yBAAyB,CAAC,EAAE,OAAO,CAAA;KAAE,CAAM,GAC9D,mCAAmC,CAAC,CAAC,CAAC;IASzC;;;OAGG;IACH,4BAA4B,CAAC,CAAC,EAC5B,UAAU,EAAE,eAAe,EAC3B,UAAU,EAAE,sCAAsC,EAClD,QAAQ,EAAE,MAAM,CAAC,GAChB,mCAAmC,CAAC,CAAC,CAAC;IAWzC,gFAAgF;IAChF,6BAA6B,CAAC,CAAC,EAAE,UAAU,EAAE,eAAe,EAC1D,UAAU,EAAE,QAAQ,CAAC;QAAE,cAAc,EAAE,MAAM,CAAA;KAAE,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,GAAG,mCAAmC,CAAC,CAAC,CAAC;IA8E9G,sFAAsF;IACtF,UAAU,CAAC,KAAK,EAAE,YAAY,GAAG,gBAAgB,EAAE;IAuCnD,KAAK,CAAC,KAAK,EAAE,YAAY,GAAG,aAAa,EAAE;IAuC3C,OAAO,CAAC,KAAK,EAAE,mBAAmB,GAAG,cAAc;IA8EnD,MAAM,IAAI,gBAAgB;CA2X3B"}
|
package/lib/store.js
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
1
|
import { createHash, randomUUID } from 'node:crypto';
|
|
2
|
+
import { AttentionTuningLedger } from './attention-tuning.js';
|
|
3
|
+
import { AttentionOwnerQuestionLedger } from './attention-owner-questions.js';
|
|
4
|
+
import { NotificationOutcomeLedger } from './notification-outcomes.js';
|
|
2
5
|
import { isAbsolute, resolve } from 'node:path';
|
|
3
6
|
import { evaluationSchemaVersion, openEvaluationDatabase } from './sqlite.js';
|
|
4
7
|
import { deliveryStatuses, executionStatuses, objectiveStatuses, outcomeSourceKinds, outcomeTrustLevels, } from './types.js';
|
|
@@ -254,7 +257,17 @@ export function evaluationLearningProjectionDigest(input) {
|
|
|
254
257
|
input.projection.evidenceOutcomeId ?? null,
|
|
255
258
|
]);
|
|
256
259
|
}
|
|
257
|
-
function taskSubject(scopeKey, outcomeId, references) {
|
|
260
|
+
function taskSubject(scopeKey, outcomeId, references, authority) {
|
|
261
|
+
if (authority?.trust === 'trusted' && (authority.source.kind === 'system' && authority.source.id === 'assistant-work-items-owner-proof'
|
|
262
|
+
&& authority.evaluator.id === 'assistant-work-items-owner-proof' && authority.evaluator.version === '1'
|
|
263
|
+
|| authority.source.kind === 'user-feedback' && authority.source.id === 'assistant-delivery/typed-owner-feedback'
|
|
264
|
+
&& authority.evaluator.id === 'assistant-delivery-owner-feedback' && authority.evaluator.version === '2')) {
|
|
265
|
+
const targets = references.filter(item => item.kind === 'work-item-owner-result');
|
|
266
|
+
if (targets.length === 1 && !references.some(item => ['automation-run', 'foreground-turn', 'goal-step', 'goal-outcome'].includes(item.kind))) {
|
|
267
|
+
const ref = targets[0].ref;
|
|
268
|
+
return Object.freeze({ key: JSON.stringify([scopeKey, 'work-item', ref]), kind: 'work-item', ref });
|
|
269
|
+
}
|
|
270
|
+
}
|
|
258
271
|
const typed = references.flatMap(reference => (reference.kind === 'automation-run' || reference.kind === 'foreground-turn' || reference.kind === 'goal-step' || reference.kind === 'goal-outcome'
|
|
259
272
|
? [{ kind: reference.kind, ref: reference.ref }]
|
|
260
273
|
: []));
|
|
@@ -279,6 +292,10 @@ function taskSubject(scopeKey, outcomeId, references) {
|
|
|
279
292
|
function newer(left, right) {
|
|
280
293
|
if (left.recorded_at !== right.recorded_at)
|
|
281
294
|
return left.recorded_at - right.recorded_at;
|
|
295
|
+
// Two trusted facts can share one millisecond. Random UUIDs do not encode order,
|
|
296
|
+
// so the durable append sequence decides which fact is the current revision.
|
|
297
|
+
if (left.seq !== undefined && right.seq !== undefined && left.seq !== right.seq)
|
|
298
|
+
return left.seq - right.seq;
|
|
282
299
|
return left.id === right.id ? 0 : left.id > right.id ? 1 : -1;
|
|
283
300
|
}
|
|
284
301
|
function latest(rows) {
|
|
@@ -308,7 +325,7 @@ function isAuthenticatedOwnerFeedback(row) {
|
|
|
308
325
|
&& row.evaluator_id === 'assistant-delivery-owner-feedback'
|
|
309
326
|
&& row.evaluator_version === '2'
|
|
310
327
|
&& (containsEvidence(row, 'automation-run') || containsEvidence(row, 'foreground-turn')
|
|
311
|
-
|| containsEvidence(row, 'goal-outcome'))
|
|
328
|
+
|| containsEvidence(row, 'goal-outcome') || containsEvidence(row, 'work-item-owner-result'))
|
|
312
329
|
&& containsEvidence(row, 'delivery-outbox');
|
|
313
330
|
}
|
|
314
331
|
function isTrustedVerifierReceipt(row) {
|
|
@@ -320,6 +337,38 @@ function isTrustedVerifierReceipt(row) {
|
|
|
320
337
|
&& containsEvidence(row, 'acceptance-contract')
|
|
321
338
|
&& containsEvidence(row, 'verification-receipt');
|
|
322
339
|
}
|
|
340
|
+
function workItemProductionIdentity(evidence) {
|
|
341
|
+
const matches = evidence.filter(item => item.kind === 'work-item-production-proof');
|
|
342
|
+
return matches.length === 1 && matches[0].digest !== undefined
|
|
343
|
+
? JSON.stringify([matches[0].ref, matches[0].digest]) : undefined;
|
|
344
|
+
}
|
|
345
|
+
function isAuthoritativeWorkItemTerminal(row) {
|
|
346
|
+
return row.trust === 'trusted' && row.source_kind === 'system'
|
|
347
|
+
&& row.source_id === 'assistant-work-items-owner-proof'
|
|
348
|
+
&& row.evaluator_id === 'assistant-work-items-owner-proof' && row.evaluator_version === '1'
|
|
349
|
+
&& containsEvidence(row, 'work-item-owner-result') && containsEvidence(row, 'work-item-production-proof')
|
|
350
|
+
&& row.execution_status !== 'unknown' && row.objective_status === 'unknown';
|
|
351
|
+
}
|
|
352
|
+
function latestWorkItemTerminal(rows) {
|
|
353
|
+
const revision = (row) => {
|
|
354
|
+
const reports = JSON.parse(row.evidence_json).filter(item => item.kind === 'work-item-report');
|
|
355
|
+
if (reports.length !== 1)
|
|
356
|
+
return 0;
|
|
357
|
+
const value = Number(reports[0].ref.slice(reports[0].ref.lastIndexOf('#') + 1));
|
|
358
|
+
return Number.isSafeInteger(value) && value > 0 ? value : 0;
|
|
359
|
+
};
|
|
360
|
+
const candidates = rows.filter(isAuthoritativeWorkItemTerminal).filter(row => revision(row) > 0)
|
|
361
|
+
.sort((left, right) => revision(left) - revision(right) || newer(left, right));
|
|
362
|
+
const latest = candidates.at(-1);
|
|
363
|
+
if (latest === undefined)
|
|
364
|
+
return undefined;
|
|
365
|
+
// Report order is ledger authority. Millisecond timestamps and random audit
|
|
366
|
+
// ids cannot make an older report win, and conflicting proof for one report
|
|
367
|
+
// cannot be silently resolved into a learning-eligible execution.
|
|
368
|
+
const identities = new Set(candidates.filter(row => revision(row) === revision(latest))
|
|
369
|
+
.map(row => workItemProductionIdentity(JSON.parse(row.evidence_json))));
|
|
370
|
+
return identities.size === 1 && !identities.has(undefined) ? latest : undefined;
|
|
371
|
+
}
|
|
323
372
|
function isAuthoritativeForegroundTerminal(row) {
|
|
324
373
|
return isTrustedVerifierReceipt(row)
|
|
325
374
|
&& row.source_kind === 'evaluator'
|
|
@@ -334,9 +383,18 @@ export class EvaluationStore {
|
|
|
334
383
|
#maxEvidenceRefs;
|
|
335
384
|
#maxSummaryWindowMs;
|
|
336
385
|
#defaultSummaryWindowMs;
|
|
386
|
+
/** WP22 read-only per-notification outcome canon. */
|
|
387
|
+
notifications;
|
|
388
|
+
/** WP22 slice 2 tighten-only attention tuning versions over the notification canon. */
|
|
389
|
+
attention;
|
|
390
|
+
/** WP22 slice 4 at-most-once delivery and record-only reply ledger for owner questions. */
|
|
391
|
+
attentionQuestions;
|
|
337
392
|
constructor(options) {
|
|
338
393
|
this.#database = openEvaluationDatabase(options.path);
|
|
339
394
|
this.#now = options.now ?? Date.now;
|
|
395
|
+
this.notifications = new NotificationOutcomeLedger(this.#database, this.#now);
|
|
396
|
+
this.attention = new AttentionTuningLedger(this.#database, this.#now);
|
|
397
|
+
this.attentionQuestions = new AttentionOwnerQuestionLedger(this.#database, this.#now);
|
|
340
398
|
this.#maxQueryLimit = options.maxQueryLimit ?? 100;
|
|
341
399
|
this.#maxSituationBytes = options.maxSituationBytes ?? 200;
|
|
342
400
|
this.#maxMetricsBytes = options.maxMetricsBytes ?? 4_096;
|
|
@@ -429,7 +487,7 @@ export class EvaluationStore {
|
|
|
429
487
|
const payloadHash = digest(normalized);
|
|
430
488
|
const recordedAt = timestamp(this.#now(), 'recordedAt');
|
|
431
489
|
const outcomeId = `outcome-${randomUUID()}`;
|
|
432
|
-
const verifierSubject = taskSubject(normalized.scopeKey, outcomeId, normalized.evidence);
|
|
490
|
+
const verifierSubject = taskSubject(normalized.scopeKey, outcomeId, normalized.evidence, normalized);
|
|
433
491
|
const exactEvidence = (entries, kind, ref, digestValue) => entries.some(item => (item.kind === kind && item.ref === ref
|
|
434
492
|
&& (digestValue === undefined || item.digest === digestValue)));
|
|
435
493
|
if (normalized.scopeKey !== scopeKey || normalized.situation !== claims.situation
|
|
@@ -456,7 +514,7 @@ export class EvaluationStore {
|
|
|
456
514
|
AND outcome.task_subject_ref = ? AND outcome.trust = 'trusted'
|
|
457
515
|
AND outcome.source_kind = 'evaluator' AND outcome.source_id = 'assistant-verifier'
|
|
458
516
|
AND outcome.evaluator_id = 'assistant-verifier' AND outcome.evaluator_version = '1'
|
|
459
|
-
ORDER BY outcome.recorded_at DESC, outcome.
|
|
517
|
+
ORDER BY outcome.recorded_at DESC, outcome.rowid DESC LIMIT 1
|
|
460
518
|
`).get(subject.key, subjectRef);
|
|
461
519
|
if (row === undefined) {
|
|
462
520
|
const metric = normalized.metrics;
|
|
@@ -543,7 +601,10 @@ export class EvaluationStore {
|
|
|
543
601
|
const subjectRef = generic ? subjectRefOrPrincipal : subjectKindOrRef;
|
|
544
602
|
const principalRecordId = generic ? principalRecordIdOrVersion : subjectRefOrPrincipal;
|
|
545
603
|
const version = generic ? principalVersion : principalRecordIdOrVersion;
|
|
546
|
-
const
|
|
604
|
+
const scopeKey = canonicalEvaluationScope(scope).scopeKey;
|
|
605
|
+
const subject = subjectKind === 'work-item'
|
|
606
|
+
? { key: JSON.stringify([scopeKey, 'work-item', subjectRef]) }
|
|
607
|
+
: taskSubject(scopeKey, '', [{ kind: subjectKind, ref: subjectRef }]);
|
|
547
608
|
const row = this.#database.prepare(`
|
|
548
609
|
SELECT revision.version, outcome.objective_status
|
|
549
610
|
FROM evaluation_owner_revisions revision JOIN evaluation_outcomes outcome ON outcome.id = revision.outcome_id
|
|
@@ -560,7 +621,7 @@ export class EvaluationStore {
|
|
|
560
621
|
const normalized = this.#normalize(input);
|
|
561
622
|
const payloadHash = digest(normalized);
|
|
562
623
|
const id = `outcome-${randomUUID()}`;
|
|
563
|
-
const subject = taskSubject(normalized.scopeKey, id, normalized.evidence);
|
|
624
|
+
const subject = taskSubject(normalized.scopeKey, id, normalized.evidence, normalized);
|
|
564
625
|
const recordedAt = timestamp(this.#now(), 'recordedAt');
|
|
565
626
|
const metric = normalized.metrics;
|
|
566
627
|
let committed = false;
|
|
@@ -581,7 +642,7 @@ export class EvaluationStore {
|
|
|
581
642
|
return this.ownerCommandResult(result);
|
|
582
643
|
}
|
|
583
644
|
previous = this.#database.prepare(`
|
|
584
|
-
SELECT revision.outcome_id, revision.version, outcome.objective_status
|
|
645
|
+
SELECT revision.outcome_id, revision.version, outcome.objective_status, outcome.evidence_json
|
|
585
646
|
FROM evaluation_owner_revisions revision JOIN evaluation_outcomes outcome ON outcome.id = revision.outcome_id
|
|
586
647
|
WHERE revision.subject_key = ? AND revision.lineage = ? ORDER BY revision.version DESC LIMIT 1
|
|
587
648
|
`).get(subject.key, JSON.stringify([ownerCommand.principalRecordId, ownerCommand.principalVersion]));
|
|
@@ -593,9 +654,12 @@ export class EvaluationStore {
|
|
|
593
654
|
committed = true;
|
|
594
655
|
throw new EvaluationStoreError('version-conflict', 'owner judgement changed; inspect current feedback and retry');
|
|
595
656
|
}
|
|
657
|
+
const sameWorkItemProof = subject.kind !== 'work-item' || previous === undefined
|
|
658
|
+
|| workItemProductionIdentity(JSON.parse(previous.evidence_json))
|
|
659
|
+
=== workItemProductionIdentity(normalized.evidence);
|
|
596
660
|
if (previous !== undefined && (ownerCommand.action === 'initial'
|
|
597
|
-
|| previous.objective_status === normalized.objectiveStatus)) {
|
|
598
|
-
if (previous.objective_status !== normalized.objectiveStatus) {
|
|
661
|
+
|| previous.objective_status === normalized.objectiveStatus && sameWorkItemProof)) {
|
|
662
|
+
if (previous.objective_status !== normalized.objectiveStatus || !sameWorkItemProof) {
|
|
599
663
|
this.#database.prepare(`INSERT INTO evaluation_owner_commands VALUES (?, ?, NULL, 'idempotency-conflict')`)
|
|
600
664
|
.run(ownerCommand.operationId, commandHash);
|
|
601
665
|
this.#database.exec('COMMIT');
|
|
@@ -649,9 +713,9 @@ export class EvaluationStore {
|
|
|
649
713
|
INSERT INTO evaluation_projection_outbox(
|
|
650
714
|
evaluation_id, status, attempt_count, next_attempt_at,
|
|
651
715
|
last_failure_at, last_failure_code, created_at, updated_at)
|
|
652
|
-
VALUES (?,
|
|
716
|
+
VALUES (?, ?, 0, ?, NULL, NULL, ?, ?)
|
|
653
717
|
ON CONFLICT(evaluation_id) DO NOTHING
|
|
654
|
-
`).run(winner.id, winner.recorded_at, winner.recorded_at, winner.recorded_at);
|
|
718
|
+
`).run(winner.id, winnerSubject.kind === 'work-item' ? 'not-required' : 'pending', winner.recorded_at, winner.recorded_at, winner.recorded_at);
|
|
655
719
|
this.#advanceScopeWatermark(winner.scope_key, winner.recorded_at, winnerSubject.key);
|
|
656
720
|
}
|
|
657
721
|
this.#database.exec('COMMIT');
|
|
@@ -799,7 +863,7 @@ export class EvaluationStore {
|
|
|
799
863
|
FROM evaluation_self_assessments assessment
|
|
800
864
|
JOIN evaluation_outcomes outcome ON outcome.id = assessment.outcome_id
|
|
801
865
|
WHERE assessment.scope_key = ? AND assessment.outcome_id IN (${placeholders})
|
|
802
|
-
ORDER BY assessment.occurred_at DESC, assessment.
|
|
866
|
+
ORDER BY assessment.occurred_at DESC, assessment.rowid DESC
|
|
803
867
|
`).all(scopeKey, ...outcomeIds);
|
|
804
868
|
const latest = new Map();
|
|
805
869
|
for (const row of rows) {
|
|
@@ -854,12 +918,24 @@ export class EvaluationStore {
|
|
|
854
918
|
JOIN evaluation_projection_outbox outbox ON outbox.evaluation_id = audit.id
|
|
855
919
|
WHERE task.scope_key = ? AND task.task_subject_kind = 'foreground-turn'
|
|
856
920
|
AND task.task_subject_ref = ? AND audit.trust = 'trusted'
|
|
857
|
-
ORDER BY audit.recorded_at DESC, audit.
|
|
921
|
+
ORDER BY audit.recorded_at DESC, audit.rowid DESC LIMIT 1`)
|
|
858
922
|
.get(scopeKey, inboxId);
|
|
859
923
|
if (row === undefined)
|
|
860
924
|
return undefined;
|
|
861
925
|
return this.getTaskLearningProjection(scopeInput, row.evaluation_id);
|
|
862
926
|
}
|
|
927
|
+
/** Exact work-item owner-result projection; historical system settlements stay audit-only. */
|
|
928
|
+
getWorkItemLearningProjection(scopeInput, itemIdInput) {
|
|
929
|
+
const { scopeKey } = canonicalEvaluationScope(scopeInput);
|
|
930
|
+
const itemId = boundedText(itemIdInput, 'itemId', 1_000);
|
|
931
|
+
const row = this.#database.prepare(`SELECT outbox.evaluation_id
|
|
932
|
+
FROM evaluation_task_projection_view task
|
|
933
|
+
JOIN evaluation_outcomes audit ON audit.task_subject_key = task.task_subject_key AND audit.scope_key = task.scope_key
|
|
934
|
+
JOIN evaluation_projection_outbox outbox ON outbox.evaluation_id = audit.id
|
|
935
|
+
WHERE task.scope_key = ? AND task.task_subject_kind = 'work-item' AND task.task_subject_ref = ? AND audit.trust = 'trusted'
|
|
936
|
+
ORDER BY audit.recorded_at DESC, audit.rowid DESC LIMIT 1`).get(scopeKey, itemId);
|
|
937
|
+
return row === undefined ? undefined : this.getTaskLearningProjection(scopeInput, row.evaluation_id);
|
|
938
|
+
}
|
|
863
939
|
/** Exact whole-goal lookup by the verifier assessment identity. */
|
|
864
940
|
getGoalOutcomeLearningProjection(scopeInput, assessmentIdInput) {
|
|
865
941
|
const { scopeKey } = canonicalEvaluationScope(scopeInput);
|
|
@@ -955,7 +1031,7 @@ export class EvaluationStore {
|
|
|
955
1031
|
SELECT outbox.evaluation_id FROM evaluation_projection_outbox outbox
|
|
956
1032
|
JOIN evaluation_outcomes audit ON audit.id = outbox.evaluation_id
|
|
957
1033
|
WHERE audit.task_subject_key = projection.subject_key AND audit.trust = 'trusted'
|
|
958
|
-
ORDER BY audit.recorded_at DESC, audit.
|
|
1034
|
+
ORDER BY audit.recorded_at DESC, audit.rowid DESC LIMIT 1
|
|
959
1035
|
) AS trigger_outcome_id
|
|
960
1036
|
FROM evaluation_task_projection_feed_heads head
|
|
961
1037
|
JOIN evaluation_task_projections projection ON projection.subject_key = head.subject_key
|
|
@@ -1014,6 +1090,9 @@ export class EvaluationStore {
|
|
|
1014
1090
|
withLearningWriterFence(scopeInput, fenceInput, callback, options = {}) {
|
|
1015
1091
|
const { scopeKey } = canonicalEvaluationScope(scopeInput);
|
|
1016
1092
|
const fence = this.#normalizeLearningWriterFence(fenceInput, false);
|
|
1093
|
+
if (options.requireProjectionDelivery !== false && fence.evidence.some(item => item.subjectKind === 'work-item')) {
|
|
1094
|
+
throw new EvaluationStoreError('invalid-input', 'work-item evidence is not supported by the Evolution writer fence');
|
|
1095
|
+
}
|
|
1017
1096
|
return this.#withLearningWriterFence(scopeKey, fence, callback, options);
|
|
1018
1097
|
}
|
|
1019
1098
|
/**
|
|
@@ -1271,6 +1350,7 @@ export class EvaluationStore {
|
|
|
1271
1350
|
MIN(created_at) AS oldest
|
|
1272
1351
|
FROM evaluation_projection_outbox WHERE status = 'pending'
|
|
1273
1352
|
`).get();
|
|
1353
|
+
const notifications = this.notifications.counts();
|
|
1274
1354
|
return Object.freeze({
|
|
1275
1355
|
ready: true,
|
|
1276
1356
|
schemaVersion: evaluationSchemaVersion,
|
|
@@ -1284,6 +1364,8 @@ export class EvaluationStore {
|
|
|
1284
1364
|
pendingProjections: projectionRow.pending,
|
|
1285
1365
|
retryingProjections: projectionRow.retrying ?? 0,
|
|
1286
1366
|
projectionAttempts: projectionRow.attempts,
|
|
1367
|
+
notificationOutcomes: notifications.notifications,
|
|
1368
|
+
notificationOutcomeRevisions: notifications.revisions,
|
|
1287
1369
|
...(projectionRow.oldest === null ? {} : { oldestPendingProjectionAt: projectionRow.oldest }),
|
|
1288
1370
|
...(row.latest === null ? {} : { latestOccurredAt: row.latest }),
|
|
1289
1371
|
});
|
|
@@ -1329,7 +1411,7 @@ export class EvaluationStore {
|
|
|
1329
1411
|
continue;
|
|
1330
1412
|
const current = this.#database.prepare(`
|
|
1331
1413
|
SELECT projection.scope_key, projection.primary_outcome_id, projection.updated_at,
|
|
1332
|
-
outcome.trust
|
|
1414
|
+
projection.subject_kind, outcome.trust
|
|
1333
1415
|
FROM evaluation_task_projections projection
|
|
1334
1416
|
JOIN evaluation_outcomes outcome ON outcome.id = projection.primary_outcome_id
|
|
1335
1417
|
WHERE projection.subject_key = ?
|
|
@@ -1340,15 +1422,21 @@ export class EvaluationStore {
|
|
|
1340
1422
|
INSERT INTO evaluation_projection_outbox(
|
|
1341
1423
|
evaluation_id, status, attempt_count, next_attempt_at,
|
|
1342
1424
|
last_failure_at, last_failure_code, created_at, updated_at)
|
|
1343
|
-
VALUES (?,
|
|
1425
|
+
VALUES (?, ?, 0, ?, NULL, NULL, ?, ?)
|
|
1344
1426
|
ON CONFLICT(evaluation_id) DO UPDATE SET
|
|
1345
|
-
status =
|
|
1427
|
+
status = excluded.status, attempt_count = 0,
|
|
1346
1428
|
next_attempt_at = excluded.next_attempt_at,
|
|
1347
1429
|
last_failure_at = NULL, last_failure_code = NULL,
|
|
1348
1430
|
updated_at = excluded.updated_at
|
|
1349
|
-
`).run(current.primary_outcome_id, current.updated_at, current.updated_at, current.updated_at);
|
|
1431
|
+
`).run(current.primary_outcome_id, current.subject_kind === 'work-item' ? 'not-required' : 'pending', current.updated_at, current.updated_at, current.updated_at);
|
|
1350
1432
|
this.#advanceScopeWatermark(current.scope_key, current.updated_at, subjectKey);
|
|
1351
1433
|
}
|
|
1434
|
+
// An unsupported optional consumer must never become retry work. This
|
|
1435
|
+
// also settles pending rows from an interrupted pre-release Host without
|
|
1436
|
+
// inventing an Evolution applied receipt or rewriting canonical evidence.
|
|
1437
|
+
this.#database.prepare(`UPDATE evaluation_projection_outbox SET status = 'not-required'
|
|
1438
|
+
WHERE status = 'pending' AND EXISTS (SELECT 1 FROM evaluation_outcomes outcome
|
|
1439
|
+
WHERE outcome.id = evaluation_projection_outbox.evaluation_id AND outcome.task_subject_kind = 'work-item')`).run();
|
|
1352
1440
|
this.#database.exec('COMMIT');
|
|
1353
1441
|
}
|
|
1354
1442
|
catch (error) {
|
|
@@ -1365,8 +1453,8 @@ export class EvaluationStore {
|
|
|
1365
1453
|
throw new EvaluationStoreError('not-found', 'task projection subject was not found');
|
|
1366
1454
|
}
|
|
1367
1455
|
const rows = this.#database.prepare(`
|
|
1368
|
-
SELECT * FROM evaluation_outcomes WHERE task_subject_key = ?
|
|
1369
|
-
ORDER BY recorded_at,
|
|
1456
|
+
SELECT rowid AS seq, * FROM evaluation_outcomes WHERE task_subject_key = ?
|
|
1457
|
+
ORDER BY recorded_at, rowid
|
|
1370
1458
|
`).all(subjectKey);
|
|
1371
1459
|
if (rows.length === 0)
|
|
1372
1460
|
return { learningVersionChanged: false };
|
|
@@ -1385,7 +1473,9 @@ export class EvaluationStore {
|
|
|
1385
1473
|
const terminals = rows.filter(row => isAuthoritativeAutomationTerminal(row));
|
|
1386
1474
|
execution = projection.subject_kind === 'automation-run'
|
|
1387
1475
|
? latest(terminals)
|
|
1388
|
-
:
|
|
1476
|
+
: projection.subject_kind === 'work-item'
|
|
1477
|
+
? latestWorkItemTerminal(rows)
|
|
1478
|
+
: latest(rows.filter(row => isAuthoritativeForegroundTerminal(row)));
|
|
1389
1479
|
const superseded = new Set(this.#database.prepare(`
|
|
1390
1480
|
SELECT previous_outcome_id FROM evaluation_owner_revisions WHERE subject_key = ? AND previous_outcome_id IS NOT NULL
|
|
1391
1481
|
`).all(subjectKey).map(row => row.previous_outcome_id));
|
|
@@ -1416,7 +1506,7 @@ export class EvaluationStore {
|
|
|
1416
1506
|
delivery = ownerDelivery ?? trustedDelivery ?? execution;
|
|
1417
1507
|
const rankedPrimary = rows.map(row => ({
|
|
1418
1508
|
row,
|
|
1419
|
-
rank: isAuthoritativeAutomationTerminal(row) || isAuthoritativeForegroundTerminal(row)
|
|
1509
|
+
rank: isAuthoritativeAutomationTerminal(row) || isAuthoritativeForegroundTerminal(row) || isAuthoritativeWorkItemTerminal(row)
|
|
1420
1510
|
? 4
|
|
1421
1511
|
: isAuthenticatedOwnerFeedback(row)
|
|
1422
1512
|
? 3
|
|
@@ -1430,11 +1520,17 @@ export class EvaluationStore {
|
|
|
1430
1520
|
const objectiveSituationMismatch = objective !== undefined && objective.situation !== primary.situation;
|
|
1431
1521
|
const hostRunbook = execution !== undefined && isAuthoritativeAutomationTerminal(execution)
|
|
1432
1522
|
&& /^host-runbook-v[1-9][0-9]*$/u.test(execution.evaluator_version);
|
|
1523
|
+
const workItemProofIdentity = execution === undefined ? undefined
|
|
1524
|
+
: workItemProductionIdentity(JSON.parse(execution.evidence_json));
|
|
1525
|
+
const workItemProofMatches = projection.subject_kind !== 'work-item' || objective !== undefined
|
|
1526
|
+
&& workItemProofIdentity !== undefined
|
|
1527
|
+
&& workItemProofIdentity === workItemProductionIdentity(JSON.parse(objective.evidence_json));
|
|
1433
1528
|
const learningDisposition = !objectiveConflicted
|
|
1529
|
+
&& workItemProofMatches
|
|
1434
1530
|
&& !objectiveSituationMismatch
|
|
1435
1531
|
&& objective?.trust === 'trusted'
|
|
1436
1532
|
&& (objectiveStatus === 'achieved' || objectiveStatus === 'not-achieved')
|
|
1437
|
-
&& (
|
|
1533
|
+
&& (!['automation-run', 'work-item'].includes(projection.subject_kind) || execution !== undefined)
|
|
1438
1534
|
&& !hostRunbook
|
|
1439
1535
|
? 'upsert'
|
|
1440
1536
|
: 'retract';
|
|
@@ -1505,7 +1601,7 @@ export class EvaluationStore {
|
|
|
1505
1601
|
const entries = input.evidence.map((raw, index) => {
|
|
1506
1602
|
if (typeof raw !== 'object' || raw === null || Array.isArray(raw)
|
|
1507
1603
|
|| (raw.disposition !== 'upsert' && (!allowRetract || raw.disposition !== 'retract'))
|
|
1508
|
-
|| (raw.subjectKind !== 'automation-run' && raw.subjectKind !== 'foreground-turn' && raw.subjectKind !== 'goal-step' && raw.subjectKind !== 'goal-outcome' && raw.subjectKind !== 'outcome')
|
|
1604
|
+
|| (raw.subjectKind !== 'automation-run' && raw.subjectKind !== 'foreground-turn' && raw.subjectKind !== 'goal-step' && raw.subjectKind !== 'goal-outcome' && raw.subjectKind !== 'work-item' && raw.subjectKind !== 'outcome')
|
|
1509
1605
|
|| !Number.isSafeInteger(raw.version) || raw.version < 1
|
|
1510
1606
|
|| raw.version > 1_000_000_000
|
|
1511
1607
|
|| typeof raw.digest !== 'string' || !/^[a-f\d]{64}$/u.test(raw.digest)) {
|