@seanhogg/builderforce-memory-engine 2026.9.33 → 2026.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/evermind/delta_wire.d.ts +10 -1
- package/dist/evermind/delta_wire.d.ts.map +1 -1
- package/dist/evermind/delta_wire.js +25 -3
- package/dist/evermind/delta_wire.js.map +1 -1
- package/dist/evermind/index.d.ts +4 -4
- package/dist/evermind/index.d.ts.map +1 -1
- package/dist/evermind/index.js +2 -2
- package/dist/evermind/index.js.map +1 -1
- package/dist/evermind/merge.d.ts +25 -0
- package/dist/evermind/merge.d.ts.map +1 -1
- package/dist/evermind/merge.js +30 -0
- package/dist/evermind/merge.js.map +1 -1
- package/dist/evermind/package.d.ts +9 -0
- package/dist/evermind/package.d.ts.map +1 -1
- package/dist/evermind/package.js +20 -0
- package/dist/evermind/package.js.map +1 -1
- package/dist/index.d.ts +6 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -3
- package/dist/index.js.map +1 -1
- package/dist/limbic/index.d.ts +3 -0
- package/dist/limbic/index.d.ts.map +1 -1
- package/dist/limbic/index.js +3 -0
- package/dist/limbic/index.js.map +1 -1
- package/dist/limbic/limbic_task.d.ts +22 -0
- package/dist/limbic/limbic_task.d.ts.map +1 -0
- package/dist/limbic/limbic_task.js +45 -0
- package/dist/limbic/limbic_task.js.map +1 -0
- package/dist/limbic/limbic_trainer.d.ts +17 -30
- package/dist/limbic/limbic_trainer.d.ts.map +1 -1
- package/dist/limbic/limbic_trainer.js +38 -144
- package/dist/limbic/limbic_trainer.js.map +1 -1
- package/dist/ml/classical/ewma.d.ts +32 -0
- package/dist/ml/classical/ewma.d.ts.map +1 -0
- package/dist/ml/classical/ewma.js +46 -0
- package/dist/ml/classical/ewma.js.map +1 -0
- package/dist/ml/classical/gbm.d.ts +42 -0
- package/dist/ml/classical/gbm.d.ts.map +1 -0
- package/dist/ml/classical/gbm.js +116 -0
- package/dist/ml/classical/gbm.js.map +1 -0
- package/dist/ml/classical/index.d.ts +48 -0
- package/dist/ml/classical/index.d.ts.map +1 -0
- package/dist/ml/classical/index.js +45 -0
- package/dist/ml/classical/index.js.map +1 -0
- package/dist/ml/classical/isolation_forest.d.ts +37 -0
- package/dist/ml/classical/isolation_forest.d.ts.map +1 -0
- package/dist/ml/classical/isolation_forest.js +72 -0
- package/dist/ml/classical/isolation_forest.js.map +1 -0
- package/dist/ml/classical/kmeans.d.ts +31 -0
- package/dist/ml/classical/kmeans.d.ts.map +1 -0
- package/dist/ml/classical/kmeans.js +93 -0
- package/dist/ml/classical/kmeans.js.map +1 -0
- package/dist/ml/classical/knn.d.ts +48 -0
- package/dist/ml/classical/knn.d.ts.map +1 -0
- package/dist/ml/classical/knn.js +68 -0
- package/dist/ml/classical/knn.js.map +1 -0
- package/dist/ml/classical/linear.d.ts +29 -0
- package/dist/ml/classical/linear.d.ts.map +1 -0
- package/dist/ml/classical/linear.js +114 -0
- package/dist/ml/classical/linear.js.map +1 -0
- package/dist/ml/classical/logistic.d.ts +39 -0
- package/dist/ml/classical/logistic.d.ts.map +1 -0
- package/dist/ml/classical/logistic.js +103 -0
- package/dist/ml/classical/logistic.js.map +1 -0
- package/dist/ml/classical/math.d.ts +26 -0
- package/dist/ml/classical/math.d.ts.map +1 -0
- package/dist/ml/classical/math.js +90 -0
- package/dist/ml/classical/math.js.map +1 -0
- package/dist/ml/classical/naive_bayes.d.ts +24 -0
- package/dist/ml/classical/naive_bayes.d.ts.map +1 -0
- package/dist/ml/classical/naive_bayes.js +67 -0
- package/dist/ml/classical/naive_bayes.js.map +1 -0
- package/dist/ml/classical/robust_z.d.ts +30 -0
- package/dist/ml/classical/robust_z.d.ts.map +1 -0
- package/dist/ml/classical/robust_z.js +46 -0
- package/dist/ml/classical/robust_z.js.map +1 -0
- package/dist/ml/dense_head.d.ts +33 -0
- package/dist/ml/dense_head.d.ts.map +1 -0
- package/dist/ml/dense_head.js +140 -0
- package/dist/ml/dense_head.js.map +1 -0
- package/dist/ml/features.d.ts +25 -0
- package/dist/ml/features.d.ts.map +1 -0
- package/dist/ml/features.js +46 -0
- package/dist/ml/features.js.map +1 -0
- package/dist/ml/federate.d.ts +70 -0
- package/dist/ml/federate.d.ts.map +1 -0
- package/dist/ml/federate.js +122 -0
- package/dist/ml/federate.js.map +1 -0
- package/dist/ml/head_trainer.d.ts +76 -0
- package/dist/ml/head_trainer.d.ts.map +1 -0
- package/dist/ml/head_trainer.js +182 -0
- package/dist/ml/head_trainer.js.map +1 -0
- package/dist/ml/index.d.ts +23 -0
- package/dist/ml/index.d.ts.map +1 -0
- package/dist/ml/index.js +11 -0
- package/dist/ml/index.js.map +1 -0
- package/dist/ml/registry.d.ts +20 -0
- package/dist/ml/registry.d.ts.map +1 -0
- package/dist/ml/registry.js +43 -0
- package/dist/ml/registry.js.map +1 -0
- package/dist/ml/run.d.ts +94 -0
- package/dist/ml/run.d.ts.map +1 -0
- package/dist/ml/run.js +130 -0
- package/dist/ml/run.js.map +1 -0
- package/dist/ml/tasks/experience.d.ts +90 -0
- package/dist/ml/tasks/experience.d.ts.map +1 -0
- package/dist/ml/tasks/experience.js +156 -0
- package/dist/ml/tasks/experience.js.map +1 -0
- package/dist/ml/tasks/index.d.ts +10 -0
- package/dist/ml/tasks/index.d.ts.map +1 -0
- package/dist/ml/tasks/index.js +28 -0
- package/dist/ml/tasks/index.js.map +1 -0
- package/dist/ml/tasks/local_success.d.ts +55 -0
- package/dist/ml/tasks/local_success.d.ts.map +1 -0
- package/dist/ml/tasks/local_success.js +77 -0
- package/dist/ml/tasks/local_success.js.map +1 -0
- package/dist/ml/tasks/run_anomaly.d.ts +51 -0
- package/dist/ml/tasks/run_anomaly.d.ts.map +1 -0
- package/dist/ml/tasks/run_anomaly.js +60 -0
- package/dist/ml/tasks/run_anomaly.js.map +1 -0
- package/dist/ml/train.d.ts +64 -0
- package/dist/ml/train.d.ts.map +1 -0
- package/dist/ml/train.js +101 -0
- package/dist/ml/train.js.map +1 -0
- package/dist/ml/types.d.ts +112 -0
- package/dist/ml/types.d.ts.map +1 -0
- package/dist/ml/types.js +23 -0
- package/dist/ml/types.js.map +1 -0
- package/dist/moe/moe_model.d.ts.map +1 -1
- package/dist/moe/moe_model.js +1 -16
- package/dist/moe/moe_model.js.map +1 -1
- package/dist/training/cpu_ops.d.ts +2 -0
- package/dist/training/cpu_ops.d.ts.map +1 -1
- package/dist/training/cpu_ops.js +16 -0
- package/dist/training/cpu_ops.js.map +1 -1
- package/package.json +1 -1
- package/src/evermind/delta_wire.ts +148 -114
- package/src/evermind/index.ts +35 -32
- package/src/evermind/merge.ts +53 -0
- package/src/evermind/package.ts +488 -464
- package/src/index.ts +15 -1
- package/src/limbic/index.ts +5 -0
- package/src/limbic/limbic_task.ts +62 -0
- package/src/limbic/limbic_trainer.ts +91 -258
- package/src/ml/classical/ewma.ts +68 -0
- package/src/ml/classical/gbm.ts +143 -0
- package/src/ml/classical/index.ts +76 -0
- package/src/ml/classical/isolation_forest.ts +88 -0
- package/src/ml/classical/kmeans.ts +102 -0
- package/src/ml/classical/knn.ts +98 -0
- package/src/ml/classical/linear.ts +119 -0
- package/src/ml/classical/logistic.ts +125 -0
- package/src/ml/classical/math.ts +93 -0
- package/src/ml/classical/naive_bayes.ts +77 -0
- package/src/ml/classical/robust_z.ts +65 -0
- package/src/ml/dense_head.ts +157 -0
- package/src/ml/features.ts +61 -0
- package/src/ml/federate.ts +160 -0
- package/src/ml/head_trainer.ts +268 -0
- package/src/ml/index.ts +57 -0
- package/src/ml/registry.ts +44 -0
- package/src/ml/run.ts +178 -0
- package/src/ml/tasks/experience.ts +214 -0
- package/src/ml/tasks/index.ts +30 -0
- package/src/ml/tasks/local_success.ts +118 -0
- package/src/ml/tasks/run_anomaly.ts +90 -0
- package/src/ml/train.ts +136 -0
- package/src/ml/types.ts +132 -0
- package/src/moe/moe_model.ts +1 -13
- package/src/training/cpu_ops.ts +141 -127
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ml/tasks/experience.ts — `experience.quality` and `experience.dedupe`, and the
|
|
3
|
+
* opt-in curation step that applies them in front of the ONE adaptation recipe
|
|
4
|
+
* (`evermind/adapt.ts`), so the private model learns from good, varied examples
|
|
5
|
+
* instead of every transcript. Curation only drops documents; it never rewrites one.
|
|
6
|
+
*/
|
|
7
|
+
import { type ExperienceDocument } from "../../experience/corpus.js";
|
|
8
|
+
import type { Episode, Run, Skill } from "../../experience/types.js";
|
|
9
|
+
import { type TextEmbedder } from "../features.js";
|
|
10
|
+
import { type MlRunContext } from "../run.js";
|
|
11
|
+
import type { MlTrainExample } from "../train.js";
|
|
12
|
+
import type { MlPrediction, MlPredictor, MlTask } from "../types.js";
|
|
13
|
+
/** One learnable procedure plus the outcome fields that say how well it went. */
|
|
14
|
+
export interface ExperienceQualityInput {
|
|
15
|
+
doc: ExperienceDocument;
|
|
16
|
+
/** Steps in the procedure. */
|
|
17
|
+
steps: number;
|
|
18
|
+
/** Runs of the skill (0 for an uncompiled demonstration). */
|
|
19
|
+
runs: number;
|
|
20
|
+
succeeded: number;
|
|
21
|
+
failed: number;
|
|
22
|
+
}
|
|
23
|
+
export interface ExperienceQuality {
|
|
24
|
+
label: "low" | "high";
|
|
25
|
+
/** P(high). */
|
|
26
|
+
pHigh: number;
|
|
27
|
+
}
|
|
28
|
+
export declare const EXPERIENCE_QUALITY_TASK: MlTask<ExperienceQualityInput, ExperienceQuality>;
|
|
29
|
+
/** Quality inputs for every learnable document, with run outcomes joined by skill id. */
|
|
30
|
+
export declare function experienceQualityInputs(episodes: Episode[], skills: Skill[], runs?: readonly Run[]): ExperienceQualityInput[];
|
|
31
|
+
/** The training label of a quality input: 1 (high) when most finished runs succeeded, 0 when not, null with no runs. */
|
|
32
|
+
export declare function experienceQualityLabel(i: ExperienceQualityInput): number | null;
|
|
33
|
+
/** Labelled examples for `experience.quality` — only the inputs that have run outcomes. */
|
|
34
|
+
export declare function experienceQualityExamples(inputs: readonly ExperienceQualityInput[]): MlTrainExample<ExperienceQualityInput>[];
|
|
35
|
+
export interface ExperienceDuplicate {
|
|
36
|
+
/** Index (in keep order) of the nearest already-kept document, or -1 when none. */
|
|
37
|
+
nearest: number;
|
|
38
|
+
similarity: number;
|
|
39
|
+
duplicate: boolean;
|
|
40
|
+
}
|
|
41
|
+
/** Cosine similarity at or above which two procedures count as one. */
|
|
42
|
+
export declare const DEFAULT_DEDUPE_THRESHOLD = 0.97;
|
|
43
|
+
export declare const EXPERIENCE_DEDUPE_TASK: MlTask<ExperienceDocument, ExperienceDuplicate>;
|
|
44
|
+
/**
|
|
45
|
+
* The `experience.dedupe` predictor: a cosine nearest-neighbour index of the
|
|
46
|
+
* documents kept so far. `scores = [nearestIndex, similarity, duplicate ? 1 : 0]`.
|
|
47
|
+
*/
|
|
48
|
+
export declare class DedupeIndex implements MlPredictor {
|
|
49
|
+
readonly threshold: number;
|
|
50
|
+
private readonly _knn;
|
|
51
|
+
constructor(threshold?: number);
|
|
52
|
+
get size(): number;
|
|
53
|
+
add(x: Float32Array): void;
|
|
54
|
+
predict(x: Float32Array): MlPrediction;
|
|
55
|
+
}
|
|
56
|
+
export interface CurationOptions {
|
|
57
|
+
/** Embeds each document once (the person's CPU Evermind: `EvermindTextEmbedder`). */
|
|
58
|
+
embedder: TextEmbedder;
|
|
59
|
+
/** Cosine similarity at or above which a document duplicates a kept one. Default {@link DEFAULT_DEDUPE_THRESHOLD}. */
|
|
60
|
+
dedupeThreshold?: number;
|
|
61
|
+
/**
|
|
62
|
+
* Quality filter through the runner. A document whose P(high) is below `minPHigh`
|
|
63
|
+
* (default 0.5) is dropped; one the runner cannot score (no trained engine) is kept.
|
|
64
|
+
*/
|
|
65
|
+
quality?: {
|
|
66
|
+
ctx: MlRunContext;
|
|
67
|
+
input: (doc: ExperienceDocument) => ExperienceQualityInput;
|
|
68
|
+
minPHigh?: number;
|
|
69
|
+
};
|
|
70
|
+
/** Document ids to leave out (e.g. those whose runs `run.anomaly` flagged). */
|
|
71
|
+
exclude?: ReadonlySet<string>;
|
|
72
|
+
}
|
|
73
|
+
export interface CurationResult {
|
|
74
|
+
kept: ExperienceDocument[];
|
|
75
|
+
dropped: {
|
|
76
|
+
id: string;
|
|
77
|
+
reason: "duplicate" | "low-quality" | "excluded";
|
|
78
|
+
of?: string;
|
|
79
|
+
score?: number;
|
|
80
|
+
}[];
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Curate documents before they are packed into adaptation passes: drop excluded
|
|
84
|
+
* ones, low-quality ones (when a quality filter is given), then near-duplicates of
|
|
85
|
+
* an earlier kept one. Order is preserved, and the FIRST copy wins — compiled
|
|
86
|
+
* skills come before raw demonstrations in `experienceDocuments`, so the reviewed
|
|
87
|
+
* version is the one kept.
|
|
88
|
+
*/
|
|
89
|
+
export declare function curateExperience(docs: readonly ExperienceDocument[], opts: CurationOptions): Promise<CurationResult>;
|
|
90
|
+
//# sourceMappingURL=experience.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"experience.d.ts","sourceRoot":"","sources":["../../../src/ml/tasks/experience.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,EAAuB,KAAK,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAC1F,OAAO,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,2BAA2B,CAAC;AAErE,OAAO,EAAwB,KAAK,YAAY,EAAE,MAAM,gBAAgB,CAAC;AACzE,OAAO,EAAuC,KAAK,YAAY,EAAE,MAAM,WAAW,CAAC;AACnF,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAIrE,iFAAiF;AACjF,MAAM,WAAW,sBAAsB;IACrC,GAAG,EAAE,kBAAkB,CAAC;IACxB,8BAA8B;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,6DAA6D;IAC7D,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,KAAK,GAAG,MAAM,CAAC;IACtB,eAAe;IACf,KAAK,EAAE,MAAM,CAAC;CACf;AAED,eAAO,MAAM,uBAAuB,EAAE,MAAM,CAAC,sBAAsB,EAAE,iBAAiB,CAkBrF,CAAC;AAEF,yFAAyF;AACzF,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,IAAI,GAAE,SAAS,GAAG,EAAO,GAAG,sBAAsB,EAAE,CAgBjI;AAED,wHAAwH;AACxH,wBAAgB,sBAAsB,CAAC,CAAC,EAAE,sBAAsB,GAAG,MAAM,GAAG,IAAI,CAG/E;AAED,2FAA2F;AAC3F,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,SAAS,sBAAsB,EAAE,GAAG,cAAc,CAAC,sBAAsB,CAAC,EAAE,CAK7H;AAID,MAAM,WAAW,mBAAmB;IAClC,mFAAmF;IACnF,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,uEAAuE;AACvE,eAAO,MAAM,wBAAwB,OAAO,CAAC;AAE7C,eAAO,MAAM,sBAAsB,EAAE,MAAM,CAAC,kBAAkB,EAAE,mBAAmB,CASlF,CAAC;AAEF;;;GAGG;AACH,qBAAa,WAAY,YAAW,WAAW;IAGjC,QAAQ,CAAC,SAAS,EAAE,MAAM;IAFtC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAgC;gBAEhC,SAAS,GAAE,MAAiC;IAEjE,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED,GAAG,CAAC,CAAC,EAAE,YAAY,GAAG,IAAI;IAI1B,OAAO,CAAC,CAAC,EAAE,YAAY,GAAG,YAAY;CAMvC;AAID,MAAM,WAAW,eAAe;IAC9B,qFAAqF;IACrF,QAAQ,EAAE,YAAY,CAAC;IACvB,sHAAsH;IACtH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,OAAO,CAAC,EAAE;QACR,GAAG,EAAE,YAAY,CAAC;QAClB,KAAK,EAAE,CAAC,GAAG,EAAE,kBAAkB,KAAK,sBAAsB,CAAC;QAC3D,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,OAAO,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;CAC/B;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,kBAAkB,EAAE,CAAC;IAC3B,OAAO,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,WAAW,GAAG,aAAa,GAAG,UAAU,CAAC;QAAC,EAAE,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAC1G;AAgBD;;;;;;GAMG;AACH,wBAAsB,gBAAgB,CAAC,IAAI,EAAE,SAAS,kBAAkB,EAAE,EAAE,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CAiC1H"}
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ml/tasks/experience.ts — `experience.quality` and `experience.dedupe`, and the
|
|
3
|
+
* opt-in curation step that applies them in front of the ONE adaptation recipe
|
|
4
|
+
* (`evermind/adapt.ts`), so the private model learns from good, varied examples
|
|
5
|
+
* instead of every transcript. Curation only drops documents; it never rewrites one.
|
|
6
|
+
*/
|
|
7
|
+
import { experienceDocuments } from "../../experience/corpus.js";
|
|
8
|
+
import { KnnModel } from "../classical/knn.js";
|
|
9
|
+
import { clampRatio, logScale } from "../features.js";
|
|
10
|
+
import { MlEngineUnavailableError, runMlTask } from "../run.js";
|
|
11
|
+
export const EXPERIENCE_QUALITY_TASK = {
|
|
12
|
+
kind: "classify",
|
|
13
|
+
id: "experience.quality",
|
|
14
|
+
description: "Is this procedure worth learning from (does it run, and run well)?",
|
|
15
|
+
features: {
|
|
16
|
+
text: (i) => i.doc.text,
|
|
17
|
+
numeric: (i) => [logScale(i.steps, 50), logScale(i.runs, 20), clampRatio(i.succeeded, i.runs, 1), clampRatio(i.failed, i.runs, 1)],
|
|
18
|
+
numericNames: ["steps", "runs", "successRate", "failureRate"],
|
|
19
|
+
},
|
|
20
|
+
engines: [
|
|
21
|
+
{ tier: "local", engine: "classical", algorithm: "knn" },
|
|
22
|
+
{ tier: "local", engine: "classical", algorithm: "logistic" },
|
|
23
|
+
],
|
|
24
|
+
train: { minSamples: 4 },
|
|
25
|
+
output: {
|
|
26
|
+
labels: ["low", "high"],
|
|
27
|
+
decode: (p) => ({ label: (p.scores[1] ?? 0) >= 0.5 ? "high" : "low", pHigh: p.scores[1] ?? 0 }),
|
|
28
|
+
},
|
|
29
|
+
};
|
|
30
|
+
/** Quality inputs for every learnable document, with run outcomes joined by skill id. */
|
|
31
|
+
export function experienceQualityInputs(episodes, skills, runs = []) {
|
|
32
|
+
const steps = new Map([...skills.map((s) => [s.id, s.steps.length]), ...episodes.map((e) => [e.id, e.steps.length])]);
|
|
33
|
+
const byItem = new Map();
|
|
34
|
+
for (const r of runs) {
|
|
35
|
+
if (r.status === "running")
|
|
36
|
+
continue;
|
|
37
|
+
const t = byItem.get(r.skillId) ?? { runs: 0, succeeded: 0, failed: 0 };
|
|
38
|
+
t.runs++;
|
|
39
|
+
if (r.status === "succeeded")
|
|
40
|
+
t.succeeded++;
|
|
41
|
+
if (r.status === "failed")
|
|
42
|
+
t.failed++;
|
|
43
|
+
byItem.set(r.skillId, t);
|
|
44
|
+
}
|
|
45
|
+
return experienceDocuments(episodes, skills).map((doc) => ({
|
|
46
|
+
doc,
|
|
47
|
+
steps: steps.get(doc.id) ?? 0,
|
|
48
|
+
...(byItem.get(doc.id) ?? { runs: 0, succeeded: 0, failed: 0 }),
|
|
49
|
+
}));
|
|
50
|
+
}
|
|
51
|
+
/** The training label of a quality input: 1 (high) when most finished runs succeeded, 0 when not, null with no runs. */
|
|
52
|
+
export function experienceQualityLabel(i) {
|
|
53
|
+
if (i.runs === 0)
|
|
54
|
+
return null;
|
|
55
|
+
return i.succeeded / i.runs >= 0.5 ? 1 : 0;
|
|
56
|
+
}
|
|
57
|
+
/** Labelled examples for `experience.quality` — only the inputs that have run outcomes. */
|
|
58
|
+
export function experienceQualityExamples(inputs) {
|
|
59
|
+
return inputs.flatMap((input) => {
|
|
60
|
+
const label = experienceQualityLabel(input);
|
|
61
|
+
return label == null ? [] : [{ input, label }];
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
/** Cosine similarity at or above which two procedures count as one. */
|
|
65
|
+
export const DEFAULT_DEDUPE_THRESHOLD = 0.97;
|
|
66
|
+
export const EXPERIENCE_DEDUPE_TASK = {
|
|
67
|
+
kind: "cluster",
|
|
68
|
+
id: "experience.dedupe",
|
|
69
|
+
description: "Is this procedure a near-copy of one already kept?",
|
|
70
|
+
features: { text: (d) => d.text },
|
|
71
|
+
engines: [{ tier: "local", engine: "classical", algorithm: "knn" }],
|
|
72
|
+
output: {
|
|
73
|
+
decode: (p) => ({ nearest: p.scores[0] ?? -1, similarity: p.scores[1] ?? 0, duplicate: (p.scores[2] ?? 0) === 1 }),
|
|
74
|
+
},
|
|
75
|
+
};
|
|
76
|
+
/**
|
|
77
|
+
* The `experience.dedupe` predictor: a cosine nearest-neighbour index of the
|
|
78
|
+
* documents kept so far. `scores = [nearestIndex, similarity, duplicate ? 1 : 0]`.
|
|
79
|
+
*/
|
|
80
|
+
export class DedupeIndex {
|
|
81
|
+
threshold;
|
|
82
|
+
_knn = new KnnModel(1, "cosine", 2);
|
|
83
|
+
constructor(threshold = DEFAULT_DEDUPE_THRESHOLD) {
|
|
84
|
+
this.threshold = threshold;
|
|
85
|
+
}
|
|
86
|
+
get size() {
|
|
87
|
+
return this._knn.size;
|
|
88
|
+
}
|
|
89
|
+
add(x) {
|
|
90
|
+
this._knn.add(x);
|
|
91
|
+
}
|
|
92
|
+
predict(x) {
|
|
93
|
+
const n = this._knn.nearest(x);
|
|
94
|
+
if (!n)
|
|
95
|
+
return { scores: [-1, 0, 0], confidence: 1 };
|
|
96
|
+
const dup = n.similarity >= this.threshold;
|
|
97
|
+
return { scores: [n.index, n.similarity, dup ? 1 : 0], confidence: Math.abs(n.similarity) };
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
/** Memoise an embedder for one call, so curation embeds each text once. */
|
|
101
|
+
function memoEmbedder(e) {
|
|
102
|
+
const cache = new Map();
|
|
103
|
+
return {
|
|
104
|
+
dimensions: e.dimensions,
|
|
105
|
+
fingerprint: e.fingerprint,
|
|
106
|
+
async embed(text) {
|
|
107
|
+
let v = cache.get(text);
|
|
108
|
+
if (!v)
|
|
109
|
+
cache.set(text, (v = await e.embed(text)));
|
|
110
|
+
return v;
|
|
111
|
+
},
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Curate documents before they are packed into adaptation passes: drop excluded
|
|
116
|
+
* ones, low-quality ones (when a quality filter is given), then near-duplicates of
|
|
117
|
+
* an earlier kept one. Order is preserved, and the FIRST copy wins — compiled
|
|
118
|
+
* skills come before raw demonstrations in `experienceDocuments`, so the reviewed
|
|
119
|
+
* version is the one kept.
|
|
120
|
+
*/
|
|
121
|
+
export async function curateExperience(docs, opts) {
|
|
122
|
+
const embedder = memoEmbedder(opts.embedder);
|
|
123
|
+
const index = new DedupeIndex(opts.dedupeThreshold ?? DEFAULT_DEDUPE_THRESHOLD);
|
|
124
|
+
const kept = [];
|
|
125
|
+
const dropped = [];
|
|
126
|
+
const qctx = opts.quality ? { ...opts.quality.ctx, embedder } : null;
|
|
127
|
+
for (const doc of docs) {
|
|
128
|
+
if (opts.exclude?.has(doc.id)) {
|
|
129
|
+
dropped.push({ id: doc.id, reason: "excluded" });
|
|
130
|
+
continue;
|
|
131
|
+
}
|
|
132
|
+
if (opts.quality && qctx) {
|
|
133
|
+
try {
|
|
134
|
+
const r = await runMlTask(EXPERIENCE_QUALITY_TASK.id, opts.quality.input(doc), qctx);
|
|
135
|
+
if (r.output.pHigh < (opts.quality.minPHigh ?? 0.5)) {
|
|
136
|
+
dropped.push({ id: doc.id, reason: "low-quality", score: r.output.pHigh });
|
|
137
|
+
continue;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
catch (error) {
|
|
141
|
+
if (!(error instanceof MlEngineUnavailableError))
|
|
142
|
+
throw error;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
const x = await embedder.embed(EXPERIENCE_DEDUPE_TASK.features.text(doc));
|
|
146
|
+
const d = EXPERIENCE_DEDUPE_TASK.output.decode(index.predict(x));
|
|
147
|
+
if (d.duplicate) {
|
|
148
|
+
dropped.push({ id: doc.id, reason: "duplicate", of: kept[d.nearest].id, score: d.similarity });
|
|
149
|
+
continue;
|
|
150
|
+
}
|
|
151
|
+
index.add(x);
|
|
152
|
+
kept.push(doc);
|
|
153
|
+
}
|
|
154
|
+
return { kept, dropped };
|
|
155
|
+
}
|
|
156
|
+
//# sourceMappingURL=experience.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"experience.js","sourceRoot":"","sources":["../../../src/ml/tasks/experience.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,EAAE,mBAAmB,EAA2B,MAAM,4BAA4B,CAAC;AAE1F,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAqB,MAAM,gBAAgB,CAAC;AACzE,OAAO,EAAE,wBAAwB,EAAE,SAAS,EAAqB,MAAM,WAAW,CAAC;AAuBnF,MAAM,CAAC,MAAM,uBAAuB,GAAsD;IACxF,IAAI,EAAE,UAAU;IAChB,EAAE,EAAE,oBAAoB;IACxB,WAAW,EAAE,oEAAoE;IACjF,QAAQ,EAAE;QACR,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI;QACvB,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAClI,YAAY,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,aAAa,CAAC;KAC9D;IACD,OAAO,EAAE;QACP,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,KAAK,EAAE;QACxD,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,UAAU,EAAE;KAC9D;IACD,KAAK,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE;IACxB,MAAM,EAAE;QACN,MAAM,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC;QACvB,MAAM,EAAE,CAAC,CAAe,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;KAC9G;CACF,CAAC;AAEF,yFAAyF;AACzF,MAAM,UAAU,uBAAuB,CAAC,QAAmB,EAAE,MAAe,EAAE,OAAuB,EAAE;IACrG,MAAM,KAAK,GAAG,IAAI,GAAG,CAAiB,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,CAAU,CAAC,EAAE,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,CAAU,CAAC,CAAC,CAAC,CAAC;IACxJ,MAAM,MAAM,GAAG,IAAI,GAAG,EAA+D,CAAC;IACtF,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS;YAAE,SAAS;QACrC,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;QACxE,CAAC,CAAC,IAAI,EAAE,CAAC;QACT,IAAI,CAAC,CAAC,MAAM,KAAK,WAAW;YAAE,CAAC,CAAC,SAAS,EAAE,CAAC;QAC5C,IAAI,CAAC,CAAC,MAAM,KAAK,QAAQ;YAAE,CAAC,CAAC,MAAM,EAAE,CAAC;QACtC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IAC3B,CAAC;IACD,OAAO,mBAAmB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACzD,GAAG;QACH,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC;QAC7B,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;KAChE,CAAC,CAAC,CAAC;AACN,CAAC;AAED,wHAAwH;AACxH,MAAM,UAAU,sBAAsB,CAAC,CAAyB;IAC9D,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAC9B,OAAO,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC7C,CAAC;AAED,2FAA2F;AAC3F,MAAM,UAAU,yBAAyB,CAAC,MAAyC;IACjF,OAAO,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;QAC9B,MAAM,KAAK,GAAG,sBAAsB,CAAC,KAAK,CAAC,CAAC;QAC5C,OAAO,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;AACL,CAAC;AAWD,uEAAuE;AACvE,MAAM,CAAC,MAAM,wBAAwB,GAAG,IAAI,CAAC;AAE7C,MAAM,CAAC,MAAM,sBAAsB,GAAoD;IACrF,IAAI,EAAE,SAAS;IACf,EAAE,EAAE,mBAAmB;IACvB,WAAW,EAAE,oDAAoD;IACjE,QAAQ,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE;IACjC,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;IACnE,MAAM,EAAE;QACN,MAAM,EAAE,CAAC,CAAe,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;KACjI;CACF,CAAC;AAEF;;;GAGG;AACH,MAAM,OAAO,WAAW;IAGD;IAFJ,IAAI,GAAG,IAAI,QAAQ,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC;IAErD,YAAqB,YAAoB,wBAAwB;QAA5C,cAAS,GAAT,SAAS,CAAmC;IAAG,CAAC;IAErE,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;IACxB,CAAC;IAED,GAAG,CAAC,CAAe;QACjB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACnB,CAAC;IAED,OAAO,CAAC,CAAe;QACrB,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC/B,IAAI,CAAC,CAAC;YAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC;QACrD,MAAM,GAAG,GAAG,CAAC,CAAC,UAAU,IAAI,IAAI,CAAC,SAAS,CAAC;QAC3C,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC;IAC9F,CAAC;CACF;AA2BD,2EAA2E;AAC3E,SAAS,YAAY,CAAC,CAAe;IACnC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAwB,CAAC;IAC9C,OAAO;QACL,UAAU,EAAE,CAAC,CAAC,UAAU;QACxB,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,KAAK,CAAC,KAAK,CAAC,IAAY;YACtB,IAAI,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACxB,IAAI,CAAC,CAAC;gBAAE,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACnD,OAAO,CAAC,CAAC;QACX,CAAC;KACF,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,IAAmC,EAAE,IAAqB;IAC/F,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC7C,MAAM,KAAK,GAAG,IAAI,WAAW,CAAC,IAAI,CAAC,eAAe,IAAI,wBAAwB,CAAC,CAAC;IAChF,MAAM,IAAI,GAAyB,EAAE,CAAC;IACtC,MAAM,OAAO,GAA8B,EAAE,CAAC;IAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IAErE,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;YAC9B,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;YACjD,SAAS;QACX,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,EAAE,CAAC;YACzB,IAAI,CAAC;gBACH,MAAM,CAAC,GAAG,MAAM,SAAS,CAAoB,uBAAuB,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;gBACxG,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,GAAG,CAAC,EAAE,CAAC;oBACpD,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;oBAC3E,SAAS;gBACX,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,CAAC,KAAK,YAAY,wBAAwB,CAAC;oBAAE,MAAM,KAAK,CAAC;YAChE,CAAC;QACH,CAAC;QACD,MAAM,CAAC,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,sBAAsB,CAAC,QAAQ,CAAC,IAAK,CAAC,GAAG,CAAC,CAAC,CAAC;QAC3E,MAAM,CAAC,GAAG,sBAAsB,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QACjE,IAAI,CAAC,CAAC,SAAS,EAAE,CAAC;YAChB,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,OAAO,CAAE,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;YAChG,SAAS;QACX,CAAC;QACD,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACb,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACjB,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;AAC3B,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { MlTaskRegistry } from "../registry.js";
|
|
2
|
+
import type { MlTask } from "../types.js";
|
|
3
|
+
/** The built-in rows, in catalog order. */
|
|
4
|
+
export declare const BUILTIN_ML_TASKS: readonly MlTask<any, any>[];
|
|
5
|
+
/** A registry holding every built-in row (add host rows with `register`). */
|
|
6
|
+
export declare function createMlRegistry(): MlTaskRegistry;
|
|
7
|
+
export * from "./experience.js";
|
|
8
|
+
export * from "./run_anomaly.js";
|
|
9
|
+
export * from "./local_success.js";
|
|
10
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/ml/tasks/index.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAK1C,2CAA2C;AAC3C,eAAO,MAAM,gBAAgB,EAAE,SAAS,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,EAMvD,CAAC;AAEF,6EAA6E;AAC7E,wBAAgB,gBAAgB,IAAI,cAAc,CAIjD;AAED,cAAc,iBAAiB,CAAC;AAChC,cAAc,kBAAkB,CAAC;AACjC,cAAc,oBAAoB,CAAC"}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ml/tasks — the built-in catalog rows (docs/ml-task-catalog.md §4), in order.
|
|
3
|
+
* `limbic.affect` is defined by limbic/ itself; this only lists it.
|
|
4
|
+
*/
|
|
5
|
+
import { LIMBIC_AFFECT_TASK } from "../../limbic/limbic_task.js";
|
|
6
|
+
import { MlTaskRegistry } from "../registry.js";
|
|
7
|
+
import { EXPERIENCE_DEDUPE_TASK, EXPERIENCE_QUALITY_TASK } from "./experience.js";
|
|
8
|
+
import { LOCAL_SUCCESS_TASK } from "./local_success.js";
|
|
9
|
+
import { RUN_ANOMALY_TASK } from "./run_anomaly.js";
|
|
10
|
+
/** The built-in rows, in catalog order. */
|
|
11
|
+
export const BUILTIN_ML_TASKS = [
|
|
12
|
+
LIMBIC_AFFECT_TASK,
|
|
13
|
+
EXPERIENCE_QUALITY_TASK,
|
|
14
|
+
EXPERIENCE_DEDUPE_TASK,
|
|
15
|
+
RUN_ANOMALY_TASK,
|
|
16
|
+
LOCAL_SUCCESS_TASK,
|
|
17
|
+
];
|
|
18
|
+
/** A registry holding every built-in row (add host rows with `register`). */
|
|
19
|
+
export function createMlRegistry() {
|
|
20
|
+
const r = new MlTaskRegistry();
|
|
21
|
+
for (const t of BUILTIN_ML_TASKS)
|
|
22
|
+
r.register(t);
|
|
23
|
+
return r;
|
|
24
|
+
}
|
|
25
|
+
export * from "./experience.js";
|
|
26
|
+
export * from "./run_anomaly.js";
|
|
27
|
+
export * from "./local_success.js";
|
|
28
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/ml/tasks/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,OAAO,EAAE,kBAAkB,EAAE,MAAM,6BAA6B,CAAC;AACjE,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAEhD,OAAO,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,iBAAiB,CAAC;AAClF,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAEpD,2CAA2C;AAC3C,MAAM,CAAC,MAAM,gBAAgB,GAAgC;IAC3D,kBAAkB;IAClB,uBAAuB;IACvB,sBAAsB;IACtB,gBAAgB;IAChB,kBAAkB;CACnB,CAAC;AAEF,6EAA6E;AAC7E,MAAM,UAAU,gBAAgB;IAC9B,MAAM,CAAC,GAAG,IAAI,cAAc,EAAE,CAAC;IAC/B,KAAK,MAAM,CAAC,IAAI,gBAAgB;QAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IAChD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,cAAc,iBAAiB,CAAC;AAChC,cAAc,kBAAkB,CAAC;AACjC,cAAc,oBAAoB,CAAC"}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import type { MlTrainExample } from "../train.js";
|
|
2
|
+
import type { MlTask } from "../types.js";
|
|
3
|
+
/** What the router knows about a turn before it runs, plus the local model's served context. */
|
|
4
|
+
export interface LocalTurnFeatures {
|
|
5
|
+
promptTokens: number;
|
|
6
|
+
/** Tokens of context (history, files, memory) the turn carries. */
|
|
7
|
+
contextTokens: number;
|
|
8
|
+
/** The local model's served context window, in tokens. */
|
|
9
|
+
servedContextTokens: number;
|
|
10
|
+
/** Tools advertised to the turn. */
|
|
11
|
+
tools: number;
|
|
12
|
+
/** Position of the turn in its conversation (0 = first). */
|
|
13
|
+
turnIndex: number;
|
|
14
|
+
attachments?: number;
|
|
15
|
+
/** Recent local success rate for this person/workload in [0, 1], when known. */
|
|
16
|
+
priorLocalSuccessRate?: number;
|
|
17
|
+
/** The turn is mostly code (edits, diffs, stack traces). */
|
|
18
|
+
codeHeavy?: boolean;
|
|
19
|
+
}
|
|
20
|
+
export declare const LOCAL_SUCCESS_FEATURES: readonly ["promptTokens", "contextTokens", "contextFill", "overflows", "tools", "turnIndex", "attachments", "priorLocalSuccessRate", "codeHeavy"];
|
|
21
|
+
/** Fixed scales — the same on every machine, so the weights federate. */
|
|
22
|
+
export declare function localSuccessFeatures(t: LocalTurnFeatures): number[];
|
|
23
|
+
export interface LocalSuccess {
|
|
24
|
+
succeeds: boolean;
|
|
25
|
+
/** P(the local model succeeds). */
|
|
26
|
+
probability: number;
|
|
27
|
+
}
|
|
28
|
+
export declare const LOCAL_SUCCESS_TASK: MlTask<LocalTurnFeatures, LocalSuccess>;
|
|
29
|
+
export type TurnResult = "succeeded" | "failed";
|
|
30
|
+
/** How one routed turn actually went. */
|
|
31
|
+
export interface LocalSuccessOutcome {
|
|
32
|
+
turn: LocalTurnFeatures;
|
|
33
|
+
/** The local model's result, when it ran. */
|
|
34
|
+
local?: TurnResult;
|
|
35
|
+
/** The cloud's result, when it ran. */
|
|
36
|
+
cloud?: TurnResult;
|
|
37
|
+
/** The local model ran first and the turn was handed to the cloud. */
|
|
38
|
+
fellBack?: boolean;
|
|
39
|
+
}
|
|
40
|
+
/** Weight of a cloud failure on a turn the local model never tried — weak evidence the turn was hard. */
|
|
41
|
+
export declare const CLOUD_ONLY_FAILURE_WEIGHT = 0.5;
|
|
42
|
+
/**
|
|
43
|
+
* The training label of one outcome, or null when it says nothing about the local model.
|
|
44
|
+
* - local ran: its result (1 succeeded / 0 failed);
|
|
45
|
+
* - fell back to the cloud: 0 — the local model could not finish it;
|
|
46
|
+
* - cloud only, and the cloud failed: 0 at {@link CLOUD_ONLY_FAILURE_WEIGHT};
|
|
47
|
+
* - cloud only, and the cloud succeeded: no evidence about local → null.
|
|
48
|
+
*/
|
|
49
|
+
export declare function localSuccessLabel(o: LocalSuccessOutcome): {
|
|
50
|
+
label: 0 | 1;
|
|
51
|
+
weight: number;
|
|
52
|
+
} | null;
|
|
53
|
+
/** Labelled examples from routed outcomes (local AND cloud), oldest first. */
|
|
54
|
+
export declare function localSuccessExamples(outcomes: readonly LocalSuccessOutcome[]): MlTrainExample<LocalTurnFeatures>[];
|
|
55
|
+
//# sourceMappingURL=local_success.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"local_success.d.ts","sourceRoot":"","sources":["../../../src/ml/tasks/local_success.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,KAAK,EAAgB,MAAM,EAAE,MAAM,aAAa,CAAC;AAExD,gGAAgG;AAChG,MAAM,WAAW,iBAAiB;IAChC,YAAY,EAAE,MAAM,CAAC;IACrB,mEAAmE;IACnE,aAAa,EAAE,MAAM,CAAC;IACtB,0DAA0D;IAC1D,mBAAmB,EAAE,MAAM,CAAC;IAC5B,oCAAoC;IACpC,KAAK,EAAE,MAAM,CAAC;IACd,4DAA4D;IAC5D,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gFAAgF;IAChF,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,4DAA4D;IAC5D,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED,eAAO,MAAM,sBAAsB,mJAUzB,CAAC;AAEX,yEAAyE;AACzE,wBAAgB,oBAAoB,CAAC,CAAC,EAAE,iBAAiB,GAAG,MAAM,EAAE,CAanE;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,OAAO,CAAC;IAClB,mCAAmC;IACnC,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,eAAO,MAAM,kBAAkB,EAAE,MAAM,CAAC,iBAAiB,EAAE,YAAY,CActE,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG,WAAW,GAAG,QAAQ,CAAC;AAEhD,yCAAyC;AACzC,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,iBAAiB,CAAC;IACxB,6CAA6C;IAC7C,KAAK,CAAC,EAAE,UAAU,CAAC;IACnB,uCAAuC;IACvC,KAAK,CAAC,EAAE,UAAU,CAAC;IACnB,sEAAsE;IACtE,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED,yGAAyG;AACzG,eAAO,MAAM,yBAAyB,MAAM,CAAC;AAE7C;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,mBAAmB,GAAG;IAAE,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAKjG;AAED,8EAA8E;AAC9E,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,SAAS,mBAAmB,EAAE,GAAG,cAAc,CAAC,iBAAiB,CAAC,EAAE,CAKlH"}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ml/tasks/local_success.ts — `run.local-success`: will this turn succeed on the
|
|
3
|
+
* local model? Exported for the Hybrid router (Builderforce.ai), which keeps a turn
|
|
4
|
+
* local when it can and sends it to the cloud when it can't.
|
|
5
|
+
*
|
|
6
|
+
* It LEARNS FROM THE CLOUD (docs/ml-task-catalog.md §6): in Hybrid, a turn that
|
|
7
|
+
* fell back to — or was also served by — the cloud is a training label for the
|
|
8
|
+
* local predictor, alongside local outcomes. See {@link localSuccessLabel}.
|
|
9
|
+
*/
|
|
10
|
+
import { clampRatio, logScale } from "../features.js";
|
|
11
|
+
export const LOCAL_SUCCESS_FEATURES = [
|
|
12
|
+
"promptTokens",
|
|
13
|
+
"contextTokens",
|
|
14
|
+
"contextFill",
|
|
15
|
+
"overflows",
|
|
16
|
+
"tools",
|
|
17
|
+
"turnIndex",
|
|
18
|
+
"attachments",
|
|
19
|
+
"priorLocalSuccessRate",
|
|
20
|
+
"codeHeavy",
|
|
21
|
+
];
|
|
22
|
+
/** Fixed scales — the same on every machine, so the weights federate. */
|
|
23
|
+
export function localSuccessFeatures(t) {
|
|
24
|
+
const fill = clampRatio(t.promptTokens + t.contextTokens, t.servedContextTokens, 4);
|
|
25
|
+
return [
|
|
26
|
+
logScale(t.promptTokens, 100_000),
|
|
27
|
+
logScale(t.contextTokens, 100_000),
|
|
28
|
+
fill,
|
|
29
|
+
fill > 1 ? 1 : 0,
|
|
30
|
+
logScale(t.tools, 64),
|
|
31
|
+
logScale(t.turnIndex, 100),
|
|
32
|
+
logScale(t.attachments ?? 0, 16),
|
|
33
|
+
Math.max(0, Math.min(1, t.priorLocalSuccessRate ?? 0.5)),
|
|
34
|
+
t.codeHeavy ? 1 : 0,
|
|
35
|
+
];
|
|
36
|
+
}
|
|
37
|
+
export const LOCAL_SUCCESS_TASK = {
|
|
38
|
+
kind: "classify",
|
|
39
|
+
id: "run.local-success",
|
|
40
|
+
description: "Will this turn succeed on the local model?",
|
|
41
|
+
features: { numeric: localSuccessFeatures, numericNames: LOCAL_SUCCESS_FEATURES },
|
|
42
|
+
engines: [
|
|
43
|
+
{ tier: "local", engine: "classical", algorithm: "logistic" },
|
|
44
|
+
{ tier: "local", engine: "classical", algorithm: "gbm-small" },
|
|
45
|
+
],
|
|
46
|
+
train: { minSamples: 8 },
|
|
47
|
+
output: {
|
|
48
|
+
labels: ["fails", "succeeds"],
|
|
49
|
+
decode: (p) => ({ succeeds: (p.scores[1] ?? 0) >= 0.5, probability: p.scores[1] ?? 0 }),
|
|
50
|
+
},
|
|
51
|
+
};
|
|
52
|
+
/** Weight of a cloud failure on a turn the local model never tried — weak evidence the turn was hard. */
|
|
53
|
+
export const CLOUD_ONLY_FAILURE_WEIGHT = 0.5;
|
|
54
|
+
/**
|
|
55
|
+
* The training label of one outcome, or null when it says nothing about the local model.
|
|
56
|
+
* - local ran: its result (1 succeeded / 0 failed);
|
|
57
|
+
* - fell back to the cloud: 0 — the local model could not finish it;
|
|
58
|
+
* - cloud only, and the cloud failed: 0 at {@link CLOUD_ONLY_FAILURE_WEIGHT};
|
|
59
|
+
* - cloud only, and the cloud succeeded: no evidence about local → null.
|
|
60
|
+
*/
|
|
61
|
+
export function localSuccessLabel(o) {
|
|
62
|
+
if (o.local)
|
|
63
|
+
return { label: o.local === "succeeded" ? 1 : 0, weight: 1 };
|
|
64
|
+
if (o.fellBack)
|
|
65
|
+
return { label: 0, weight: 1 };
|
|
66
|
+
if (o.cloud === "failed")
|
|
67
|
+
return { label: 0, weight: CLOUD_ONLY_FAILURE_WEIGHT };
|
|
68
|
+
return null;
|
|
69
|
+
}
|
|
70
|
+
/** Labelled examples from routed outcomes (local AND cloud), oldest first. */
|
|
71
|
+
export function localSuccessExamples(outcomes) {
|
|
72
|
+
return outcomes.flatMap((o) => {
|
|
73
|
+
const l = localSuccessLabel(o);
|
|
74
|
+
return l ? [{ input: o.turn, label: l.label, weight: l.weight }] : [];
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
//# sourceMappingURL=local_success.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"local_success.js","sourceRoot":"","sources":["../../../src/ml/tasks/local_success.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAsBtD,MAAM,CAAC,MAAM,sBAAsB,GAAG;IACpC,cAAc;IACd,eAAe;IACf,aAAa;IACb,WAAW;IACX,OAAO;IACP,WAAW;IACX,aAAa;IACb,uBAAuB;IACvB,WAAW;CACH,CAAC;AAEX,yEAAyE;AACzE,MAAM,UAAU,oBAAoB,CAAC,CAAoB;IACvD,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,aAAa,EAAE,CAAC,CAAC,mBAAmB,EAAE,CAAC,CAAC,CAAC;IACpF,OAAO;QACL,QAAQ,CAAC,CAAC,CAAC,YAAY,EAAE,OAAO,CAAC;QACjC,QAAQ,CAAC,CAAC,CAAC,aAAa,EAAE,OAAO,CAAC;QAClC,IAAI;QACJ,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAChB,QAAQ,CAAC,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC;QACrB,QAAQ,CAAC,CAAC,CAAC,SAAS,EAAE,GAAG,CAAC;QAC1B,QAAQ,CAAC,CAAC,CAAC,WAAW,IAAI,CAAC,EAAE,EAAE,CAAC;QAChC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,qBAAqB,IAAI,GAAG,CAAC,CAAC;QACxD,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;KACpB,CAAC;AACJ,CAAC;AAQD,MAAM,CAAC,MAAM,kBAAkB,GAA4C;IACzE,IAAI,EAAE,UAAU;IAChB,EAAE,EAAE,mBAAmB;IACvB,WAAW,EAAE,4CAA4C;IACzD,QAAQ,EAAE,EAAE,OAAO,EAAE,oBAAoB,EAAE,YAAY,EAAE,sBAAsB,EAAE;IACjF,OAAO,EAAE;QACP,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,UAAU,EAAE;QAC7D,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,WAAW,EAAE;KAC/D;IACD,KAAK,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE;IACxB,MAAM,EAAE;QACN,MAAM,EAAE,CAAC,OAAO,EAAE,UAAU,CAAC;QAC7B,MAAM,EAAE,CAAC,CAAe,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,GAAG,EAAE,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;KACtG;CACF,CAAC;AAeF,yGAAyG;AACzG,MAAM,CAAC,MAAM,yBAAyB,GAAG,GAAG,CAAC;AAE7C;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB,CAAC,CAAsB;IACtD,IAAI,CAAC,CAAC,KAAK;QAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,KAAK,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;IAC1E,IAAI,CAAC,CAAC,QAAQ;QAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;IAC/C,IAAI,CAAC,CAAC,KAAK,KAAK,QAAQ;QAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,yBAAyB,EAAE,CAAC;IACjF,OAAO,IAAI,CAAC;AACd,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,oBAAoB,CAAC,QAAwC;IAC3E,OAAO,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE;QAC5B,MAAM,CAAC,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAC;QAC/B,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACxE,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ml/tasks/run_anomaly.ts — `run.anomaly`: a run that cost, took or failed far
|
|
3
|
+
* outside its normal range. Flags runs to leave out of training and to surface as
|
|
4
|
+
* Insights.
|
|
5
|
+
*
|
|
6
|
+
* Two detectors, one answer: robust z (median/MAD over the whole history — "far
|
|
7
|
+
* from normal") and EWMA (the recent trend — "drifting away from lately"). The
|
|
8
|
+
* score is the larger of the two, so either one can flag.
|
|
9
|
+
*/
|
|
10
|
+
import type { Run } from "../../experience/types.js";
|
|
11
|
+
import { EwmaModel, type EwmaOptions } from "../classical/ewma.js";
|
|
12
|
+
import { RobustZModel, type RobustZOptions } from "../classical/robust_z.js";
|
|
13
|
+
import type { MlPrediction, MlPredictor, MlSample, MlTask } from "../types.js";
|
|
14
|
+
/** What a run cost and how it went — the host supplies cost; the rest is in the run log. */
|
|
15
|
+
export interface RunMetrics {
|
|
16
|
+
costUsd: number;
|
|
17
|
+
durationMs: number;
|
|
18
|
+
toolFailures: number;
|
|
19
|
+
}
|
|
20
|
+
export interface RunAnomaly {
|
|
21
|
+
anomalous: boolean;
|
|
22
|
+
/** Normalised score: ≥ 1 is anomalous. */
|
|
23
|
+
score: number;
|
|
24
|
+
}
|
|
25
|
+
export declare const RUN_ANOMALY_METRICS: readonly ["cost", "duration", "toolFailures"];
|
|
26
|
+
/** Fixed log scales (a $10 run, a one-hour run, 50 failures map to 1). */
|
|
27
|
+
export declare function runAnomalyFeatures(m: RunMetrics): number[];
|
|
28
|
+
/** Robust z over the history, combined with EWMA over its recent trend. */
|
|
29
|
+
export declare class RunAnomalyDetector implements MlPredictor {
|
|
30
|
+
readonly population: RobustZModel;
|
|
31
|
+
readonly trend: EwmaModel;
|
|
32
|
+
constructor(population: RobustZModel, trend: EwmaModel);
|
|
33
|
+
/** Which metrics are out of range, per detector (z / threshold, ≥ 1 = out of range). */
|
|
34
|
+
explain(x: ArrayLike<number>): {
|
|
35
|
+
metric: (typeof RUN_ANOMALY_METRICS)[number];
|
|
36
|
+
population: number;
|
|
37
|
+
trend: number;
|
|
38
|
+
}[];
|
|
39
|
+
/** Fold a finished, non-anomalous run into the trend. */
|
|
40
|
+
observe(x: ArrayLike<number>): void;
|
|
41
|
+
predict(x: Float32Array): MlPrediction;
|
|
42
|
+
}
|
|
43
|
+
/** Fit both detectors on a chronological history of runs (as feature vectors). */
|
|
44
|
+
export declare function fitRunAnomaly(samples: readonly MlSample[], opts?: {
|
|
45
|
+
robustZ?: RobustZOptions;
|
|
46
|
+
ewma?: EwmaOptions;
|
|
47
|
+
}): RunAnomalyDetector;
|
|
48
|
+
export declare const RUN_ANOMALY_TASK: MlTask<RunMetrics, RunAnomaly>;
|
|
49
|
+
/** Duration and tool-failure count of an experience run, with the host's cost. */
|
|
50
|
+
export declare function experienceRunMetrics(run: Run, costUsd?: number): RunMetrics;
|
|
51
|
+
//# sourceMappingURL=run_anomaly.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run_anomaly.d.ts","sourceRoot":"","sources":["../../../src/ml/tasks/run_anomaly.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,2BAA2B,CAAC;AACrD,OAAO,EAAE,SAAS,EAAW,KAAK,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAC5E,OAAO,EAAiC,YAAY,EAAE,KAAK,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAE5G,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAE/E,4FAA4F;AAC5F,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,UAAU;IACzB,SAAS,EAAE,OAAO,CAAC;IACnB,0CAA0C;IAC1C,KAAK,EAAE,MAAM,CAAC;CACf;AAED,eAAO,MAAM,mBAAmB,+CAAgD,CAAC;AAEjF,0EAA0E;AAC1E,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,UAAU,GAAG,MAAM,EAAE,CAE1D;AAED,2EAA2E;AAC3E,qBAAa,kBAAmB,YAAW,WAAW;IAElD,QAAQ,CAAC,UAAU,EAAE,YAAY;IACjC,QAAQ,CAAC,KAAK,EAAE,SAAS;gBADhB,UAAU,EAAE,YAAY,EACxB,KAAK,EAAE,SAAS;IAG3B,wFAAwF;IACxF,OAAO,CAAC,CAAC,EAAE,SAAS,CAAC,MAAM,CAAC,GAAG;QAAE,MAAM,EAAE,CAAC,OAAO,mBAAmB,CAAC,CAAC,MAAM,CAAC,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE;IAUpH,yDAAyD;IACzD,OAAO,CAAC,CAAC,EAAE,SAAS,CAAC,MAAM,CAAC,GAAG,IAAI;IAInC,OAAO,CAAC,CAAC,EAAE,YAAY,GAAG,YAAY;CAIvC;AAED,kFAAkF;AAClF,wBAAgB,aAAa,CAAC,OAAO,EAAE,SAAS,QAAQ,EAAE,EAAE,IAAI,GAAE;IAAE,OAAO,CAAC,EAAE,cAAc,CAAC;IAAC,IAAI,CAAC,EAAE,WAAW,CAAA;CAAO,GAAG,kBAAkB,CAE3I;AAED,eAAO,MAAM,gBAAgB,EAAE,MAAM,CAAC,UAAU,EAAE,UAAU,CAW3D,CAAC;AAEF,kFAAkF;AAClF,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,GAAG,EAAE,OAAO,SAAI,GAAG,UAAU,CAMtE"}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { fitEwma } from "../classical/ewma.js";
|
|
2
|
+
import { anomalyConfidence, fitRobustZ } from "../classical/robust_z.js";
|
|
3
|
+
import { logScale } from "../features.js";
|
|
4
|
+
export const RUN_ANOMALY_METRICS = ["cost", "duration", "toolFailures"];
|
|
5
|
+
/** Fixed log scales (a $10 run, a one-hour run, 50 failures map to 1). */
|
|
6
|
+
export function runAnomalyFeatures(m) {
|
|
7
|
+
return [logScale(m.costUsd * 100, 1000), logScale(m.durationMs, 3_600_000), logScale(m.toolFailures, 50)];
|
|
8
|
+
}
|
|
9
|
+
/** Robust z over the history, combined with EWMA over its recent trend. */
|
|
10
|
+
export class RunAnomalyDetector {
|
|
11
|
+
population;
|
|
12
|
+
trend;
|
|
13
|
+
constructor(population, trend) {
|
|
14
|
+
this.population = population;
|
|
15
|
+
this.trend = trend;
|
|
16
|
+
}
|
|
17
|
+
/** Which metrics are out of range, per detector (z / threshold, ≥ 1 = out of range). */
|
|
18
|
+
explain(x) {
|
|
19
|
+
const pz = this.population.zScores(x);
|
|
20
|
+
const tz = this.trend.zScores(x);
|
|
21
|
+
return RUN_ANOMALY_METRICS.map((metric, i) => ({
|
|
22
|
+
metric,
|
|
23
|
+
population: pz[i] / this.population.threshold,
|
|
24
|
+
trend: tz[i] / this.trend.threshold,
|
|
25
|
+
}));
|
|
26
|
+
}
|
|
27
|
+
/** Fold a finished, non-anomalous run into the trend. */
|
|
28
|
+
observe(x) {
|
|
29
|
+
this.trend.update(x);
|
|
30
|
+
}
|
|
31
|
+
predict(x) {
|
|
32
|
+
const score = Math.max(this.population.predict(x).scores[0], this.trend.predict(x).scores[0]);
|
|
33
|
+
return { scores: [score], confidence: anomalyConfidence(score) };
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
/** Fit both detectors on a chronological history of runs (as feature vectors). */
|
|
37
|
+
export function fitRunAnomaly(samples, opts = {}) {
|
|
38
|
+
return new RunAnomalyDetector(fitRobustZ(samples, opts.robustZ), fitEwma(samples, opts.ewma));
|
|
39
|
+
}
|
|
40
|
+
export const RUN_ANOMALY_TASK = {
|
|
41
|
+
kind: "anomaly",
|
|
42
|
+
id: "run.anomaly",
|
|
43
|
+
description: "Did this run cost, take or fail far outside its normal range?",
|
|
44
|
+
features: { numeric: runAnomalyFeatures, numericNames: RUN_ANOMALY_METRICS },
|
|
45
|
+
engines: [
|
|
46
|
+
{ tier: "local", engine: "classical", algorithm: "robust-z" },
|
|
47
|
+
{ tier: "local", engine: "classical", algorithm: "ewma" },
|
|
48
|
+
],
|
|
49
|
+
train: { minSamples: 5, fit: (samples) => fitRunAnomaly(samples) },
|
|
50
|
+
output: { decode: (p) => ({ anomalous: (p.scores[0] ?? 0) >= 1, score: p.scores[0] ?? 0 }) },
|
|
51
|
+
};
|
|
52
|
+
/** Duration and tool-failure count of an experience run, with the host's cost. */
|
|
53
|
+
export function experienceRunMetrics(run, costUsd = 0) {
|
|
54
|
+
return {
|
|
55
|
+
costUsd,
|
|
56
|
+
durationMs: run.endedAt != null ? Math.max(0, run.endedAt - run.startedAt) : 0,
|
|
57
|
+
toolFailures: run.steps.filter((s) => s.outcome === "failed").length,
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
//# sourceMappingURL=run_anomaly.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run_anomaly.js","sourceRoot":"","sources":["../../../src/ml/tasks/run_anomaly.ts"],"names":[],"mappings":"AAUA,OAAO,EAAa,OAAO,EAAoB,MAAM,sBAAsB,CAAC;AAC5E,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAqC,MAAM,0BAA0B,CAAC;AAC5G,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAgB1C,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,MAAM,EAAE,UAAU,EAAE,cAAc,CAAU,CAAC;AAEjF,0EAA0E;AAC1E,MAAM,UAAU,kBAAkB,CAAC,CAAa;IAC9C,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,GAAG,GAAG,EAAE,IAAI,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,UAAU,EAAE,SAAS,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC,CAAC;AAC5G,CAAC;AAED,2EAA2E;AAC3E,MAAM,OAAO,kBAAkB;IAElB;IACA;IAFX,YACW,UAAwB,EACxB,KAAgB;QADhB,eAAU,GAAV,UAAU,CAAc;QACxB,UAAK,GAAL,KAAK,CAAW;IACxB,CAAC;IAEJ,wFAAwF;IACxF,OAAO,CAAC,CAAoB;QAC1B,MAAM,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACtC,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACjC,OAAO,mBAAmB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YAC7C,MAAM;YACN,UAAU,EAAE,EAAE,CAAC,CAAC,CAAE,GAAG,IAAI,CAAC,UAAU,CAAC,SAAS;YAC9C,KAAK,EAAE,EAAE,CAAC,CAAC,CAAE,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS;SACrC,CAAC,CAAC,CAAC;IACN,CAAC;IAED,yDAAyD;IACzD,OAAO,CAAC,CAAoB;QAC1B,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IAED,OAAO,CAAC,CAAe;QACrB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAE,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC;QAChG,OAAO,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,iBAAiB,CAAC,KAAK,CAAC,EAAE,CAAC;IACnE,CAAC;CACF;AAED,kFAAkF;AAClF,MAAM,UAAU,aAAa,CAAC,OAA4B,EAAE,OAAyD,EAAE;IACrH,OAAO,IAAI,kBAAkB,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;AAChG,CAAC;AAED,MAAM,CAAC,MAAM,gBAAgB,GAAmC;IAC9D,IAAI,EAAE,SAAS;IACf,EAAE,EAAE,aAAa;IACjB,WAAW,EAAE,+DAA+D;IAC5E,QAAQ,EAAE,EAAE,OAAO,EAAE,kBAAkB,EAAE,YAAY,EAAE,mBAAmB,EAAE;IAC5E,OAAO,EAAE;QACP,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,UAAU,EAAE;QAC7D,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,EAAE;KAC1D;IACD,KAAK,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE;IAClE,MAAM,EAAE,EAAE,MAAM,EAAE,CAAC,CAAe,EAAE,EAAE,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE;CAC3G,CAAC;AAEF,kFAAkF;AAClF,MAAM,UAAU,oBAAoB,CAAC,GAAQ,EAAE,OAAO,GAAG,CAAC;IACxD,OAAO;QACL,OAAO;QACP,UAAU,EAAE,GAAG,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,OAAO,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9E,YAAY,EAAE,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,MAAM;KACrE,CAAC;AACJ,CAAC"}
|