@themoltnet/node-red-contrib-core 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -2
- package/dist/nodes/entries-search.html +134 -0
- package/dist/nodes/entries-search.js +78 -0
- package/dist/nodes/query-utils.js +43 -0
- package/dist/nodes/tasks-list.html +174 -0
- package/dist/nodes/tasks-list.js +77 -0
- package/examples/ab-eval-with-judge.flow.json +292 -0
- package/package.json +4 -2
package/README.md
CHANGED
|
@@ -15,10 +15,11 @@ Empirically validated against **Node-RED 5.0.0** (Node 22):
|
|
|
15
15
|
package carries no private-package runtime dependency. `@themoltnet/sdk` is
|
|
16
16
|
therefore a **devDependency** (bundled, not installed at runtime).
|
|
17
17
|
- The `.html` editor files are copied to `dist/nodes/` as assets (not compiled).
|
|
18
|
-
- Two **config nodes** (`moltnet-agent`, `moltnet-runtime-profile`) and
|
|
18
|
+
- Two **config nodes** (`moltnet-agent`, `moltnet-runtime-profile`) and eight
|
|
19
19
|
**action nodes** (`moltnet-tasks-create`, `moltnet-task-get`,
|
|
20
20
|
`moltnet-task-wait`, `moltnet-workflow-status`, `moltnet-task-builder`,
|
|
21
|
-
`moltnet-task-reader`)
|
|
21
|
+
`moltnet-task-reader`, `moltnet-tasks-list`, `moltnet-entries-search`)
|
|
22
|
+
register and appear in the palette.
|
|
22
23
|
|
|
23
24
|
## Nodes
|
|
24
25
|
|
|
@@ -38,6 +39,12 @@ Empirically validated against **Node-RED 5.0.0** (Node 22):
|
|
|
38
39
|
wins). The task `input` and advanced fields come from `msg.payload`. See
|
|
39
40
|
[Building the task request](#building-the-task-request). Holds no SDK import —
|
|
40
41
|
the SDK lives only in the config node.
|
|
42
|
+
- **`moltnet-tasks-list`** (palette: _tasks: list_) — lists tasks for the
|
|
43
|
+
referenced agent's team. Supports the server task filters (`status`,
|
|
44
|
+
`statuses`, `taskTypes`, `tags`, `excludeTags`, profile/correlation/diary,
|
|
45
|
+
proposer/claimer ids, attempts, date windows, `limit`, `cursor`). Node fields
|
|
46
|
+
fill the query; an object `msg.payload` overrides them. Emits task rows on
|
|
47
|
+
`msg.payload` and pagination/query metadata on `msg.tasks`.
|
|
41
48
|
- **`moltnet-task-get`** (palette: _task: get_) — one-shot read of a task and its
|
|
42
49
|
attempts (no polling). Emits a normalized **snapshot** on `msg.payload`:
|
|
43
50
|
`{ taskId, status, terminal, accepted, acceptedAttemptN, state, attempt,
|
|
@@ -70,6 +77,13 @@ attempts, error, task }`. `state` is the accepted attempt's output artifact
|
|
|
70
77
|
The pre-computed **`outputRef`** (`{ taskId, outputCid, role }`) chains straight
|
|
71
78
|
into a downstream `task: build`'s **References from**; set an **artifact
|
|
72
79
|
kind/title** to pre-parse a JSON artifact body into `msg.result.artifactBody`.
|
|
80
|
+
- **`moltnet-entries-search`** (palette: _entries: search_) — searches diary
|
|
81
|
+
entries using the SDK hybrid search endpoint. Supports `diaryId`, `query`,
|
|
82
|
+
`tags`, `excludeTags`, `entryTypes`, `excludeSuperseded`, `limit`, `offset`,
|
|
83
|
+
and relevance/recency/importance weights. A string `msg.payload` is treated as
|
|
84
|
+
the query; an object `msg.payload` overrides node fields and may use camelCase
|
|
85
|
+
SDK keys or snake_case MCP-style keys. Emits entries on `msg.payload` and
|
|
86
|
+
search metadata on `msg.entries`.
|
|
73
87
|
|
|
74
88
|
All nodes register a long, collision-safe `type` (`moltnet-*`) but show a short
|
|
75
89
|
`paletteLabel` under the **moltnet** category, so the palette is not crowded by
|
|
@@ -173,6 +187,17 @@ the builder's context rows**, **agent output→input chaining via the reader's
|
|
|
173
187
|
freeform rubric. Runs on one daemon; see the in-flow comment for the
|
|
174
188
|
model-specialization option.
|
|
175
189
|
|
|
190
|
+
## A/B eval with judge subflow
|
|
191
|
+
|
|
192
|
+
[`examples/ab-eval-with-judge.flow.json`](./examples/ab-eval-with-judge.flow.json)
|
|
193
|
+
imports a reusable **A/B eval with judge** subflow plus a small demo tab. The
|
|
194
|
+
subflow runs `run_eval`, locally scores required fields/findings, then creates a
|
|
195
|
+
`judge_eval_attempt` task and stores per-variant scores/deltas in flow context.
|
|
196
|
+
|
|
197
|
+
Fill the `moltnet-agent` config after import. Runtime-profile config nodes are
|
|
198
|
+
included but blank: leave them blank for any eligible daemon to claim both
|
|
199
|
+
tasks, or set producer/judge profile IDs and run one daemon per profile.
|
|
200
|
+
|
|
176
201
|
## Reproducing the issue-lifecycle shape
|
|
177
202
|
|
|
178
203
|
[`examples/issue-lifecycle.flow.json`](./examples/issue-lifecycle.flow.json)
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
<script type="text/javascript">
|
|
2
|
+
RED.nodes.registerType('moltnet-entries-search', {
|
|
3
|
+
category: 'moltnet',
|
|
4
|
+
color: '#00d4c8',
|
|
5
|
+
paletteLabel: 'entries: search',
|
|
6
|
+
defaults: {
|
|
7
|
+
name: { value: '' },
|
|
8
|
+
agent: { value: '', type: 'moltnet-agent', required: true },
|
|
9
|
+
diaryId: { value: '' },
|
|
10
|
+
query: { value: '' },
|
|
11
|
+
tags: { value: '' },
|
|
12
|
+
excludeTags: { value: '' },
|
|
13
|
+
entryTypes: { value: '' },
|
|
14
|
+
excludeSuperseded: { value: 'false' },
|
|
15
|
+
limit: { value: 10, validate: RED.validators.number() },
|
|
16
|
+
offset: { value: '', validate: RED.validators.number() },
|
|
17
|
+
wRelevance: { value: 1, validate: RED.validators.number() },
|
|
18
|
+
wRecency: { value: 0, validate: RED.validators.number() },
|
|
19
|
+
wImportance: { value: 0, validate: RED.validators.number() },
|
|
20
|
+
},
|
|
21
|
+
inputs: 1,
|
|
22
|
+
outputs: 1,
|
|
23
|
+
icon: 'font-awesome/fa-search',
|
|
24
|
+
label: function () {
|
|
25
|
+
return this.name || 'entries: search';
|
|
26
|
+
},
|
|
27
|
+
});
|
|
28
|
+
</script>
|
|
29
|
+
|
|
30
|
+
<script type="text/html" data-template-name="moltnet-entries-search">
|
|
31
|
+
<div class="form-row">
|
|
32
|
+
<label for="node-input-name"><i class="fa fa-tag"></i> Name</label>
|
|
33
|
+
<input type="text" id="node-input-name" />
|
|
34
|
+
</div>
|
|
35
|
+
<div class="form-row">
|
|
36
|
+
<label for="node-input-agent"><i class="fa fa-user"></i> Agent</label>
|
|
37
|
+
<input type="text" id="node-input-agent" />
|
|
38
|
+
</div>
|
|
39
|
+
<div class="form-row">
|
|
40
|
+
<label for="node-input-diaryId"><i class="fa fa-book"></i> Diary</label>
|
|
41
|
+
<input
|
|
42
|
+
type="text"
|
|
43
|
+
id="node-input-diaryId"
|
|
44
|
+
placeholder="agent diary by default"
|
|
45
|
+
/>
|
|
46
|
+
</div>
|
|
47
|
+
<div class="form-row">
|
|
48
|
+
<label for="node-input-query"><i class="fa fa-search"></i> Query</label>
|
|
49
|
+
<input
|
|
50
|
+
type="text"
|
|
51
|
+
id="node-input-query"
|
|
52
|
+
placeholder="msg.payload overrides"
|
|
53
|
+
/>
|
|
54
|
+
</div>
|
|
55
|
+
<div class="form-row">
|
|
56
|
+
<label for="node-input-tags"><i class="fa fa-tags"></i> Tags</label>
|
|
57
|
+
<input
|
|
58
|
+
type="text"
|
|
59
|
+
id="node-input-tags"
|
|
60
|
+
placeholder="decision,scope:node-red"
|
|
61
|
+
/>
|
|
62
|
+
</div>
|
|
63
|
+
<div class="form-row">
|
|
64
|
+
<label for="node-input-excludeTags"
|
|
65
|
+
><i class="fa fa-ban"></i> Exclude</label
|
|
66
|
+
>
|
|
67
|
+
<input
|
|
68
|
+
type="text"
|
|
69
|
+
id="node-input-excludeTags"
|
|
70
|
+
placeholder="superseded,noisy"
|
|
71
|
+
/>
|
|
72
|
+
</div>
|
|
73
|
+
<div class="form-row">
|
|
74
|
+
<label for="node-input-entryTypes"
|
|
75
|
+
><i class="fa fa-archive"></i> Types</label
|
|
76
|
+
>
|
|
77
|
+
<input
|
|
78
|
+
type="text"
|
|
79
|
+
id="node-input-entryTypes"
|
|
80
|
+
placeholder="semantic,episodic"
|
|
81
|
+
/>
|
|
82
|
+
</div>
|
|
83
|
+
<div class="form-row">
|
|
84
|
+
<label for="node-input-excludeSuperseded"
|
|
85
|
+
><i class="fa fa-filter"></i> Superseded</label
|
|
86
|
+
>
|
|
87
|
+
<select id="node-input-excludeSuperseded">
|
|
88
|
+
<option value="false">Include</option>
|
|
89
|
+
<option value="true">Exclude</option>
|
|
90
|
+
</select>
|
|
91
|
+
</div>
|
|
92
|
+
<div class="form-row">
|
|
93
|
+
<label for="node-input-limit"><i class="fa fa-hashtag"></i> Limit</label>
|
|
94
|
+
<input type="number" id="node-input-limit" />
|
|
95
|
+
</div>
|
|
96
|
+
<div class="form-row">
|
|
97
|
+
<label for="node-input-offset"><i class="fa fa-forward"></i> Offset</label>
|
|
98
|
+
<input type="number" id="node-input-offset" />
|
|
99
|
+
</div>
|
|
100
|
+
<div class="form-row">
|
|
101
|
+
<label for="node-input-wRelevance"
|
|
102
|
+
><i class="fa fa-bullseye"></i> Relevance</label
|
|
103
|
+
>
|
|
104
|
+
<input type="number" step="0.1" id="node-input-wRelevance" />
|
|
105
|
+
</div>
|
|
106
|
+
<div class="form-row">
|
|
107
|
+
<label for="node-input-wRecency"
|
|
108
|
+
><i class="fa fa-clock-o"></i> Recency</label
|
|
109
|
+
>
|
|
110
|
+
<input type="number" step="0.1" id="node-input-wRecency" />
|
|
111
|
+
</div>
|
|
112
|
+
<div class="form-row">
|
|
113
|
+
<label for="node-input-wImportance"
|
|
114
|
+
><i class="fa fa-star"></i> Importance</label
|
|
115
|
+
>
|
|
116
|
+
<input type="number" step="0.1" id="node-input-wImportance" />
|
|
117
|
+
</div>
|
|
118
|
+
</script>
|
|
119
|
+
|
|
120
|
+
<script type="text/html" data-help-name="moltnet-entries-search">
|
|
121
|
+
<p>
|
|
122
|
+
Searches MoltNet diary entries. The configured diary is used first, then the
|
|
123
|
+
agent's diary; set <code>msg.payload.diaryId</code> to override it.
|
|
124
|
+
</p>
|
|
125
|
+
<p>
|
|
126
|
+
A string <code>msg.payload</code> is treated as the search query. An object
|
|
127
|
+
<code>msg.payload</code> overrides node fields and may use either camelCase
|
|
128
|
+
SDK keys or snake_case MCP-style keys.
|
|
129
|
+
</p>
|
|
130
|
+
<p>
|
|
131
|
+
Emits search result items on <code>msg.payload</code> and metadata on
|
|
132
|
+
<code>msg.entries = { total, query }</code>.
|
|
133
|
+
</p>
|
|
134
|
+
</script>
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import { a as nonEmpty, c as positiveInt, i as finiteNumber, n as compact, o as nonNegativeInt, r as csv, s as normalizeAliases, t as bool } from "./query-utils.js";
|
|
2
|
+
//#region src/nodes/entries-search.ts
|
|
3
|
+
var init = (RED) => {
|
|
4
|
+
function EntriesSearchNode(def) {
|
|
5
|
+
RED.nodes.createNode(this, def);
|
|
6
|
+
const agentNode = def.agent ? RED.nodes.getNode(def.agent) : null;
|
|
7
|
+
this.on("input", (msg, send, done) => {
|
|
8
|
+
const run = async () => {
|
|
9
|
+
try {
|
|
10
|
+
if (!agentNode || typeof agentNode.getAgent !== "function") throw new Error("entries-search: no moltnet-agent configured");
|
|
11
|
+
const body = buildSearchBody(def, msg, agentNode.diaryId);
|
|
12
|
+
if (!body.query) throw new Error("entries-search: query is required");
|
|
13
|
+
this.status({
|
|
14
|
+
fill: "blue",
|
|
15
|
+
shape: "dot",
|
|
16
|
+
text: "searching…"
|
|
17
|
+
});
|
|
18
|
+
const result = await (await agentNode.getAgent()).entries.search(body);
|
|
19
|
+
const out = RED.util.cloneMessage(msg);
|
|
20
|
+
out.payload = result.results;
|
|
21
|
+
out.entries = {
|
|
22
|
+
total: result.total,
|
|
23
|
+
query: body
|
|
24
|
+
};
|
|
25
|
+
this.status({
|
|
26
|
+
fill: "green",
|
|
27
|
+
shape: "dot",
|
|
28
|
+
text: `${result.results.length} entr${result.results.length === 1 ? "y" : "ies"}`
|
|
29
|
+
});
|
|
30
|
+
send(out);
|
|
31
|
+
done();
|
|
32
|
+
} catch (err) {
|
|
33
|
+
this.status({
|
|
34
|
+
fill: "red",
|
|
35
|
+
shape: "ring",
|
|
36
|
+
text: "error"
|
|
37
|
+
});
|
|
38
|
+
done(err instanceof Error ? err : new Error(String(err)));
|
|
39
|
+
}
|
|
40
|
+
};
|
|
41
|
+
run();
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
RED.nodes.registerType("moltnet-entries-search", EntriesSearchNode);
|
|
45
|
+
};
|
|
46
|
+
function buildSearchBody(def, msg, agentDiaryId) {
|
|
47
|
+
const configured = {
|
|
48
|
+
diaryId: nonEmpty(def.diaryId) ?? nonEmpty(agentDiaryId),
|
|
49
|
+
query: nonEmpty(def.query),
|
|
50
|
+
tags: csv(def.tags),
|
|
51
|
+
excludeTags: csv(def.excludeTags),
|
|
52
|
+
entryTypes: csv(def.entryTypes),
|
|
53
|
+
excludeSuperseded: bool(def.excludeSuperseded),
|
|
54
|
+
limit: positiveInt(def.limit),
|
|
55
|
+
offset: nonNegativeInt(def.offset),
|
|
56
|
+
wRelevance: finiteNumber(def.wRelevance),
|
|
57
|
+
wRecency: finiteNumber(def.wRecency),
|
|
58
|
+
wImportance: finiteNumber(def.wImportance)
|
|
59
|
+
};
|
|
60
|
+
const payload = msg.payload && typeof msg.payload === "object" ? normalizePayload(msg.payload) : typeof msg.payload === "string" ? { query: msg.payload } : {};
|
|
61
|
+
return compact({
|
|
62
|
+
...configured,
|
|
63
|
+
...payload
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
function normalizePayload(payload) {
|
|
67
|
+
return normalizeAliases(payload, {
|
|
68
|
+
diary_id: "diaryId",
|
|
69
|
+
entry_types: "entryTypes",
|
|
70
|
+
exclude_superseded: "excludeSuperseded",
|
|
71
|
+
exclude_tags: "excludeTags",
|
|
72
|
+
w_importance: "wImportance",
|
|
73
|
+
w_recency: "wRecency",
|
|
74
|
+
w_relevance: "wRelevance"
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
//#endregion
|
|
78
|
+
export { init as default };
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
//#region src/nodes/query-utils.ts
|
|
2
|
+
function csv(value) {
|
|
3
|
+
if (Array.isArray(value)) {
|
|
4
|
+
const items = value.filter((item) => typeof item === "string");
|
|
5
|
+
return items.length > 0 ? items : void 0;
|
|
6
|
+
}
|
|
7
|
+
if (typeof value !== "string") return void 0;
|
|
8
|
+
const items = value.split(",").map((item) => item.trim()).filter(Boolean);
|
|
9
|
+
return items.length > 0 ? items : void 0;
|
|
10
|
+
}
|
|
11
|
+
function bool(value) {
|
|
12
|
+
if (typeof value === "boolean") return value;
|
|
13
|
+
if (value === "true") return true;
|
|
14
|
+
if (value === "false") return false;
|
|
15
|
+
}
|
|
16
|
+
function positiveInt(value) {
|
|
17
|
+
const n = typeof value === "number" ? value : Number(value);
|
|
18
|
+
return Number.isInteger(n) && n > 0 ? n : void 0;
|
|
19
|
+
}
|
|
20
|
+
function nonNegativeInt(value) {
|
|
21
|
+
const n = typeof value === "number" ? value : Number(value);
|
|
22
|
+
return Number.isInteger(n) && n >= 0 ? n : void 0;
|
|
23
|
+
}
|
|
24
|
+
function finiteNumber(value) {
|
|
25
|
+
const n = typeof value === "number" ? value : Number(value);
|
|
26
|
+
return Number.isFinite(n) ? n : void 0;
|
|
27
|
+
}
|
|
28
|
+
function nonEmpty(value) {
|
|
29
|
+
return typeof value === "string" && value.trim() ? value.trim() : void 0;
|
|
30
|
+
}
|
|
31
|
+
function compact(value) {
|
|
32
|
+
return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== void 0));
|
|
33
|
+
}
|
|
34
|
+
function normalizeAliases(payload, aliases) {
|
|
35
|
+
const normalized = { ...payload };
|
|
36
|
+
for (const [alias, canonical] of Object.entries(aliases)) if (normalized[alias] !== void 0 && normalized[canonical] === void 0) {
|
|
37
|
+
normalized[canonical] = normalized[alias];
|
|
38
|
+
delete normalized[alias];
|
|
39
|
+
}
|
|
40
|
+
return normalized;
|
|
41
|
+
}
|
|
42
|
+
//#endregion
|
|
43
|
+
export { nonEmpty as a, positiveInt as c, finiteNumber as i, compact as n, nonNegativeInt as o, csv as r, normalizeAliases as s, bool as t };
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
<script type="text/javascript">
|
|
2
|
+
RED.nodes.registerType('moltnet-tasks-list', {
|
|
3
|
+
category: 'moltnet',
|
|
4
|
+
color: '#00d4c8',
|
|
5
|
+
paletteLabel: 'tasks: list',
|
|
6
|
+
defaults: {
|
|
7
|
+
name: { value: '' },
|
|
8
|
+
agent: { value: '', type: 'moltnet-agent', required: true },
|
|
9
|
+
status: { value: '' },
|
|
10
|
+
statusList: { value: '' },
|
|
11
|
+
taskTypes: { value: '' },
|
|
12
|
+
tags: { value: '' },
|
|
13
|
+
excludeTags: { value: '' },
|
|
14
|
+
profileId: { value: '' },
|
|
15
|
+
correlationId: { value: '' },
|
|
16
|
+
diaryId: { value: '' },
|
|
17
|
+
proposedByAgentId: { value: '' },
|
|
18
|
+
proposedByHumanId: { value: '' },
|
|
19
|
+
claimedByAgentId: { value: '' },
|
|
20
|
+
hasAttempts: { value: '' },
|
|
21
|
+
queuedAfter: { value: '' },
|
|
22
|
+
queuedBefore: { value: '' },
|
|
23
|
+
completedAfter: { value: '' },
|
|
24
|
+
completedBefore: { value: '' },
|
|
25
|
+
limit: { value: 20, validate: RED.validators.number() },
|
|
26
|
+
cursor: { value: '' },
|
|
27
|
+
},
|
|
28
|
+
inputs: 1,
|
|
29
|
+
outputs: 1,
|
|
30
|
+
icon: 'font-awesome/fa-list',
|
|
31
|
+
label: function () {
|
|
32
|
+
return this.name || 'tasks: list';
|
|
33
|
+
},
|
|
34
|
+
});
|
|
35
|
+
</script>
|
|
36
|
+
|
|
37
|
+
<script type="text/html" data-template-name="moltnet-tasks-list">
|
|
38
|
+
<div class="form-row">
|
|
39
|
+
<label for="node-input-name"><i class="fa fa-tag"></i> Name</label>
|
|
40
|
+
<input type="text" id="node-input-name" />
|
|
41
|
+
</div>
|
|
42
|
+
<div class="form-row">
|
|
43
|
+
<label for="node-input-agent"><i class="fa fa-user"></i> Agent</label>
|
|
44
|
+
<input type="text" id="node-input-agent" />
|
|
45
|
+
</div>
|
|
46
|
+
<div class="form-row">
|
|
47
|
+
<label for="node-input-status"><i class="fa fa-check"></i> Status</label>
|
|
48
|
+
<input type="text" id="node-input-status" placeholder="queued" />
|
|
49
|
+
</div>
|
|
50
|
+
<div class="form-row">
|
|
51
|
+
<label for="node-input-statusList"
|
|
52
|
+
><i class="fa fa-check-square-o"></i> Statuses</label
|
|
53
|
+
>
|
|
54
|
+
<input
|
|
55
|
+
type="text"
|
|
56
|
+
id="node-input-statusList"
|
|
57
|
+
placeholder="waiting,queued"
|
|
58
|
+
/>
|
|
59
|
+
</div>
|
|
60
|
+
<div class="form-row">
|
|
61
|
+
<label for="node-input-taskTypes"><i class="fa fa-cubes"></i> Types</label>
|
|
62
|
+
<input
|
|
63
|
+
type="text"
|
|
64
|
+
id="node-input-taskTypes"
|
|
65
|
+
placeholder="fulfill_brief,render_pack"
|
|
66
|
+
/>
|
|
67
|
+
</div>
|
|
68
|
+
<div class="form-row">
|
|
69
|
+
<label for="node-input-tags"><i class="fa fa-tags"></i> Tags</label>
|
|
70
|
+
<input type="text" id="node-input-tags" placeholder="issue,triage" />
|
|
71
|
+
</div>
|
|
72
|
+
<div class="form-row">
|
|
73
|
+
<label for="node-input-excludeTags"
|
|
74
|
+
><i class="fa fa-ban"></i> Exclude</label
|
|
75
|
+
>
|
|
76
|
+
<input type="text" id="node-input-excludeTags" placeholder="archived" />
|
|
77
|
+
</div>
|
|
78
|
+
<div class="form-row">
|
|
79
|
+
<label for="node-input-profileId"
|
|
80
|
+
><i class="fa fa-id-card"></i> Profile</label
|
|
81
|
+
>
|
|
82
|
+
<input type="text" id="node-input-profileId" />
|
|
83
|
+
</div>
|
|
84
|
+
<div class="form-row">
|
|
85
|
+
<label for="node-input-correlationId"
|
|
86
|
+
><i class="fa fa-link"></i> Correlation</label
|
|
87
|
+
>
|
|
88
|
+
<input
|
|
89
|
+
type="text"
|
|
90
|
+
id="node-input-correlationId"
|
|
91
|
+
placeholder="msg.payload overrides"
|
|
92
|
+
/>
|
|
93
|
+
</div>
|
|
94
|
+
<div class="form-row">
|
|
95
|
+
<label for="node-input-diaryId"><i class="fa fa-book"></i> Diary</label>
|
|
96
|
+
<input type="text" id="node-input-diaryId" />
|
|
97
|
+
</div>
|
|
98
|
+
<div class="form-row">
|
|
99
|
+
<label for="node-input-proposedByAgentId"
|
|
100
|
+
><i class="fa fa-user-circle"></i> Proposed agent</label
|
|
101
|
+
>
|
|
102
|
+
<input type="text" id="node-input-proposedByAgentId" />
|
|
103
|
+
</div>
|
|
104
|
+
<div class="form-row">
|
|
105
|
+
<label for="node-input-proposedByHumanId"
|
|
106
|
+
><i class="fa fa-user"></i> Proposed human</label
|
|
107
|
+
>
|
|
108
|
+
<input type="text" id="node-input-proposedByHumanId" />
|
|
109
|
+
</div>
|
|
110
|
+
<div class="form-row">
|
|
111
|
+
<label for="node-input-claimedByAgentId"
|
|
112
|
+
><i class="fa fa-hand-paper-o"></i> Claimed agent</label
|
|
113
|
+
>
|
|
114
|
+
<input type="text" id="node-input-claimedByAgentId" />
|
|
115
|
+
</div>
|
|
116
|
+
<div class="form-row">
|
|
117
|
+
<label for="node-input-hasAttempts"
|
|
118
|
+
><i class="fa fa-history"></i> Attempts</label
|
|
119
|
+
>
|
|
120
|
+
<select id="node-input-hasAttempts">
|
|
121
|
+
<option value="">Any</option>
|
|
122
|
+
<option value="true">Has attempts</option>
|
|
123
|
+
<option value="false">No attempts</option>
|
|
124
|
+
</select>
|
|
125
|
+
</div>
|
|
126
|
+
<div class="form-row">
|
|
127
|
+
<label for="node-input-queuedAfter"
|
|
128
|
+
><i class="fa fa-clock-o"></i> Queued after</label
|
|
129
|
+
>
|
|
130
|
+
<input
|
|
131
|
+
type="text"
|
|
132
|
+
id="node-input-queuedAfter"
|
|
133
|
+
placeholder="2026-06-25T00:00:00Z"
|
|
134
|
+
/>
|
|
135
|
+
</div>
|
|
136
|
+
<div class="form-row">
|
|
137
|
+
<label for="node-input-queuedBefore"
|
|
138
|
+
><i class="fa fa-clock-o"></i> Queued before</label
|
|
139
|
+
>
|
|
140
|
+
<input type="text" id="node-input-queuedBefore" />
|
|
141
|
+
</div>
|
|
142
|
+
<div class="form-row">
|
|
143
|
+
<label for="node-input-completedAfter"
|
|
144
|
+
><i class="fa fa-calendar-check-o"></i> Done after</label
|
|
145
|
+
>
|
|
146
|
+
<input type="text" id="node-input-completedAfter" />
|
|
147
|
+
</div>
|
|
148
|
+
<div class="form-row">
|
|
149
|
+
<label for="node-input-completedBefore"
|
|
150
|
+
><i class="fa fa-calendar-check-o"></i> Done before</label
|
|
151
|
+
>
|
|
152
|
+
<input type="text" id="node-input-completedBefore" />
|
|
153
|
+
</div>
|
|
154
|
+
<div class="form-row">
|
|
155
|
+
<label for="node-input-limit"><i class="fa fa-hashtag"></i> Limit</label>
|
|
156
|
+
<input type="number" id="node-input-limit" />
|
|
157
|
+
</div>
|
|
158
|
+
<div class="form-row">
|
|
159
|
+
<label for="node-input-cursor"><i class="fa fa-forward"></i> Cursor</label>
|
|
160
|
+
<input type="text" id="node-input-cursor" />
|
|
161
|
+
</div>
|
|
162
|
+
</script>
|
|
163
|
+
|
|
164
|
+
<script type="text/html" data-help-name="moltnet-tasks-list">
|
|
165
|
+
<p>
|
|
166
|
+
Lists MoltNet tasks for the configured agent's team. Node fields build the
|
|
167
|
+
query, and any object fields on <code>msg.payload</code> override them.
|
|
168
|
+
</p>
|
|
169
|
+
<p>
|
|
170
|
+
Emits the task array on <code>msg.payload</code>. Pagination metadata and
|
|
171
|
+
the final query are on <code>msg.tasks</code>:
|
|
172
|
+
<code>{ total, nextCursor, query }</code>.
|
|
173
|
+
</p>
|
|
174
|
+
</script>
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import { a as nonEmpty, c as positiveInt, n as compact, r as csv, t as bool } from "./query-utils.js";
|
|
2
|
+
//#region src/nodes/tasks-list.ts
|
|
3
|
+
var init = (RED) => {
|
|
4
|
+
function TasksListNode(def) {
|
|
5
|
+
RED.nodes.createNode(this, def);
|
|
6
|
+
const agentNode = def.agent ? RED.nodes.getNode(def.agent) : null;
|
|
7
|
+
this.on("input", (msg, send, done) => {
|
|
8
|
+
const run = async () => {
|
|
9
|
+
try {
|
|
10
|
+
if (!agentNode || typeof agentNode.getAgent !== "function") throw new Error("tasks-list: no moltnet-agent configured");
|
|
11
|
+
const teamId = agentNode.teamId;
|
|
12
|
+
if (!teamId) throw new Error("tasks-list: agent teamId is required");
|
|
13
|
+
this.status({
|
|
14
|
+
fill: "blue",
|
|
15
|
+
shape: "dot",
|
|
16
|
+
text: "loading…"
|
|
17
|
+
});
|
|
18
|
+
const agent = await agentNode.getAgent();
|
|
19
|
+
const query = buildTasksQuery(def, msg);
|
|
20
|
+
const result = await agent.tasks.list(query, { teamId });
|
|
21
|
+
const out = RED.util.cloneMessage(msg);
|
|
22
|
+
out.payload = result.items;
|
|
23
|
+
out.tasks = {
|
|
24
|
+
total: result.total,
|
|
25
|
+
nextCursor: result.nextCursor,
|
|
26
|
+
query
|
|
27
|
+
};
|
|
28
|
+
this.status({
|
|
29
|
+
fill: "green",
|
|
30
|
+
shape: "dot",
|
|
31
|
+
text: `${result.items.length} task(s)`
|
|
32
|
+
});
|
|
33
|
+
send(out);
|
|
34
|
+
done();
|
|
35
|
+
} catch (err) {
|
|
36
|
+
this.status({
|
|
37
|
+
fill: "red",
|
|
38
|
+
shape: "ring",
|
|
39
|
+
text: "error"
|
|
40
|
+
});
|
|
41
|
+
done(err instanceof Error ? err : new Error(String(err)));
|
|
42
|
+
}
|
|
43
|
+
};
|
|
44
|
+
run();
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
RED.nodes.registerType("moltnet-tasks-list", TasksListNode);
|
|
48
|
+
};
|
|
49
|
+
function buildTasksQuery(def, msg) {
|
|
50
|
+
const configured = {
|
|
51
|
+
status: nonEmpty(def.status),
|
|
52
|
+
statuses: csv(def.statusList),
|
|
53
|
+
taskTypes: csv(def.taskTypes),
|
|
54
|
+
tags: csv(def.tags),
|
|
55
|
+
excludeTags: csv(def.excludeTags),
|
|
56
|
+
profileId: nonEmpty(def.profileId),
|
|
57
|
+
correlationId: nonEmpty(def.correlationId),
|
|
58
|
+
diaryId: nonEmpty(def.diaryId),
|
|
59
|
+
proposedByAgentId: nonEmpty(def.proposedByAgentId),
|
|
60
|
+
proposedByHumanId: nonEmpty(def.proposedByHumanId),
|
|
61
|
+
claimedByAgentId: nonEmpty(def.claimedByAgentId),
|
|
62
|
+
hasAttempts: bool(def.hasAttempts),
|
|
63
|
+
queuedAfter: nonEmpty(def.queuedAfter),
|
|
64
|
+
queuedBefore: nonEmpty(def.queuedBefore),
|
|
65
|
+
completedAfter: nonEmpty(def.completedAfter),
|
|
66
|
+
completedBefore: nonEmpty(def.completedBefore),
|
|
67
|
+
limit: positiveInt(def.limit),
|
|
68
|
+
cursor: nonEmpty(def.cursor)
|
|
69
|
+
};
|
|
70
|
+
const payload = msg.payload && typeof msg.payload === "object" ? msg.payload : {};
|
|
71
|
+
return compact({
|
|
72
|
+
...configured,
|
|
73
|
+
...payload
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
//#endregion
|
|
77
|
+
export { init as default };
|
|
@@ -0,0 +1,292 @@
|
|
|
1
|
+
[
|
|
2
|
+
{
|
|
3
|
+
"category": "moltnet",
|
|
4
|
+
"color": "#D7F7C2",
|
|
5
|
+
"env": [],
|
|
6
|
+
"icon": "font-awesome/fa-balance-scale",
|
|
7
|
+
"id": "subflow_ab_eval_with_judge",
|
|
8
|
+
"in": [
|
|
9
|
+
{
|
|
10
|
+
"wires": [
|
|
11
|
+
{
|
|
12
|
+
"id": "sf_ab_build_run_eval"
|
|
13
|
+
}
|
|
14
|
+
],
|
|
15
|
+
"x": 40,
|
|
16
|
+
"y": 120
|
|
17
|
+
}
|
|
18
|
+
],
|
|
19
|
+
"info": "Generic run_eval -> local score -> judge_eval_attempt pipeline. Parent flow supplies msg.evalScenario, msg.evalVariantLabel, msg.evalSkillContext, msg.evalJudgeCriteria, and msg.correlationId. Configure the agent/runtime-profile nodes after import, then run matching agent daemons.",
|
|
20
|
+
"meta": {},
|
|
21
|
+
"name": "A/B eval with judge",
|
|
22
|
+
"out": [
|
|
23
|
+
{
|
|
24
|
+
"wires": [
|
|
25
|
+
{
|
|
26
|
+
"id": "sf_ab_store_delta",
|
|
27
|
+
"port": 0
|
|
28
|
+
}
|
|
29
|
+
],
|
|
30
|
+
"x": 1280,
|
|
31
|
+
"y": 280
|
|
32
|
+
}
|
|
33
|
+
],
|
|
34
|
+
"type": "subflow"
|
|
35
|
+
},
|
|
36
|
+
{
|
|
37
|
+
"func": "const uuidRe = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;\nconst scenario = msg.evalScenario;\nif (!scenario || typeof scenario !== 'object') {\n node.error('Missing msg.evalScenario', msg);\n return null;\n}\nconst variantLabel = msg.evalVariantLabel || msg.payload?.variantLabel || 'skill-rubric-v1';\nconst correlationId = msg.correlationId || msg.evalGroupCorrelationId || msg.payload?.correlationId;\nif (!uuidRe.test(String(correlationId || ''))) {\n node.error('Missing valid correlationId for eval group', msg);\n return null;\n}\nconst evidence = scenario.evidence || {};\nconst expected = scenario.expected || {};\nconst basePrompt = scenario.prompt || [\n 'Run this eval scenario.',\n '',\n 'Return ONLY one valid JSON object. Do not use Markdown, headings, prose, tables, or code fences.',\n 'Use only the supplied evidence. Do not browse. Do not invent facts. Mark unknowns explicitly.',\n '',\n 'Scenario: ' + (scenario.title || scenario.id || 'unknown'),\n 'Variant: ' + variantLabel,\n '',\n 'Evidence JSON:',\n JSON.stringify(evidence, null, 2)\n].join('\\n');\nconst taskContexts = [\n { slug: 'eval-evidence', binding: 'context_inline', content: JSON.stringify(evidence, null, 2) },\n { slug: 'eval-expectations', binding: 'context_inline', content: JSON.stringify(expected, null, 2) }\n];\nif (variantLabel !== 'baseline-no-skill' && msg.evalSkillContext?.content) {\n taskContexts.unshift({\n slug: msg.evalSkillContext.slug || 'eval-skill-context',\n binding: msg.evalSkillContext.binding || 'skill',\n content: msg.evalSkillContext.content\n });\n}\nmsg.correlationId = correlationId;\nmsg.evalGroupCorrelationId = correlationId;\nmsg.evalScenario = scenario;\nmsg.evalVariantLabel = variantLabel;\nmsg.payload = {\n taskType: 'run_eval',\n title: 'Eval: ' + (scenario.id || 'scenario') + ' / ' + variantLabel,\n tags: ['eval', scenario.id || 'scenario', variantLabel],\n correlationId,\n input: {\n scenario: { prompt: basePrompt },\n variantLabel,\n execution: { mode: 'vitro', workspace: 'none' },\n context: taskContexts\n }\n};\nreturn msg;",
|
|
38
|
+
"id": "sf_ab_build_run_eval",
|
|
39
|
+
"name": "build run_eval",
|
|
40
|
+
"outputs": 1,
|
|
41
|
+
"type": "function",
|
|
42
|
+
"wires": [["sf_ab_create_run_eval"]],
|
|
43
|
+
"x": 170,
|
|
44
|
+
"y": 120,
|
|
45
|
+
"z": "subflow_ab_eval_with_judge"
|
|
46
|
+
},
|
|
47
|
+
{
|
|
48
|
+
"agent": "eval_agent_cfg",
|
|
49
|
+
"generateCorrelationId": false,
|
|
50
|
+
"id": "sf_ab_create_run_eval",
|
|
51
|
+
"maxAttempts": 1,
|
|
52
|
+
"name": "RUN EVAL producer",
|
|
53
|
+
"runtimeProfile": "eval_profile_producer",
|
|
54
|
+
"type": "moltnet-tasks-create",
|
|
55
|
+
"wires": [["sf_ab_wait_run_eval"]],
|
|
56
|
+
"x": 390,
|
|
57
|
+
"y": 120,
|
|
58
|
+
"z": "subflow_ab_eval_with_judge"
|
|
59
|
+
},
|
|
60
|
+
{
|
|
61
|
+
"agent": "eval_agent_cfg",
|
|
62
|
+
"id": "sf_ab_wait_run_eval",
|
|
63
|
+
"kinds": "",
|
|
64
|
+
"name": "wait RUN EVAL",
|
|
65
|
+
"pollIntervalSec": 5,
|
|
66
|
+
"tail": true,
|
|
67
|
+
"taskId": "",
|
|
68
|
+
"timeoutSec": 1800,
|
|
69
|
+
"type": "moltnet-task-wait",
|
|
70
|
+
"wires": [[], ["sf_ab_stash_run_eval"]],
|
|
71
|
+
"x": 600,
|
|
72
|
+
"y": 120,
|
|
73
|
+
"z": "subflow_ab_eval_with_judge"
|
|
74
|
+
},
|
|
75
|
+
{
|
|
76
|
+
"func": "msg.evalTaskId = msg.payload?.taskId || msg.payload?.task?.id || msg.payload?.id || msg.taskId || null;\nmsg.evalAcceptedAttemptN = msg.payload?.acceptedAttemptN || msg.payload?.task?.acceptedAttemptN || 1;\nreturn msg;",
|
|
77
|
+
"id": "sf_ab_stash_run_eval",
|
|
78
|
+
"name": "stash producer task id",
|
|
79
|
+
"outputs": 1,
|
|
80
|
+
"type": "function",
|
|
81
|
+
"wires": [["sf_ab_read_run_eval"]],
|
|
82
|
+
"x": 820,
|
|
83
|
+
"y": 120,
|
|
84
|
+
"z": "subflow_ab_eval_with_judge"
|
|
85
|
+
},
|
|
86
|
+
{
|
|
87
|
+
"artifactKind": "",
|
|
88
|
+
"artifactTitle": "",
|
|
89
|
+
"id": "sf_ab_read_run_eval",
|
|
90
|
+
"name": "read RUN EVAL",
|
|
91
|
+
"role": "context",
|
|
92
|
+
"source": "payload",
|
|
93
|
+
"type": "moltnet-task-reader",
|
|
94
|
+
"wires": [["sf_ab_score_run_eval"]],
|
|
95
|
+
"x": 1040,
|
|
96
|
+
"y": 120,
|
|
97
|
+
"z": "subflow_ab_eval_with_judge"
|
|
98
|
+
},
|
|
99
|
+
{
|
|
100
|
+
"func": "function extractJsonObject(text) {\n if (!text || typeof text !== 'string') return null;\n const fenced = text.match(/```json\\s*([\\s\\S]*?)```/i) || text.match(/```\\s*([\\s\\S]*?)```/);\n const candidates = [];\n if (fenced) candidates.push(fenced[1]);\n const first = text.indexOf('{');\n const last = text.lastIndexOf('}');\n if (first >= 0 && last > first) candidates.push(text.slice(first, last + 1));\n for (const candidate of candidates) {\n try { return JSON.parse(candidate); } catch (_) {}\n }\n return null;\n}\nconst output = msg.payload || {};\nconst response = output.response || msg.result?.summary || '';\nconst analysis = extractJsonObject(response);\nconst expected = msg.evalScenario?.expected || {};\nconst requiredFields = msg.evalRequiredFields || ['decision', 'readinessScore0to100', 'decisionRationale', 'topActions', 'unknowns'];\nconst missingFields = analysis ? requiredFields.filter((field) => analysis[field] === undefined) : requiredFields;\nconst responseLower = response.toLowerCase();\nconst requiredFindings = expected.requiredFindings || [];\nconst missingFindings = requiredFindings.filter((item) => !responseLower.includes(String(item).toLowerCase()));\nconst forbiddenHits = (expected.forbidden || []).filter((item) => responseLower.includes(String(item).toLowerCase()));\nconst decision = analysis?.decision || null;\nconst decisionOk = decision ? (expected.acceptableDecisions || []).includes(decision) : false;\nconst checks = {\n parsedAnalysis: Boolean(analysis),\n requiredFields: missingFields.length === 0,\n decisionOk,\n noForbiddenHits: forbiddenHits.length === 0,\n requiredFindingsPresent: missingFindings.length === 0\n};\nconst score0to100 = Math.round((Object.values(checks).filter(Boolean).length / Object.keys(checks).length) * 100);\nmsg.evalProducerScore = {\n scenarioId: msg.evalScenario?.id,\n variantLabel: msg.evalVariantLabel,\n correlationId: msg.correlationId,\n taskId: msg.evalTaskId || null,\n score0to100,\n checks,\n decision,\n missingFields,\n missingFindings,\n forbiddenHits,\n analysis,\n responseRef: msg.result?.outputRef || null\n};\nreturn msg;",
|
|
101
|
+
"id": "sf_ab_score_run_eval",
|
|
102
|
+
"name": "local score",
|
|
103
|
+
"outputs": 1,
|
|
104
|
+
"type": "function",
|
|
105
|
+
"wires": [["sf_ab_build_judge_eval"]],
|
|
106
|
+
"x": 200,
|
|
107
|
+
"y": 210,
|
|
108
|
+
"z": "subflow_ab_eval_with_judge"
|
|
109
|
+
},
|
|
110
|
+
{
|
|
111
|
+
"func": "const targetTaskId = msg.evalTaskId || msg.payload?.taskId;\nconst targetAttemptN = msg.evalAcceptedAttemptN || msg.payload?.acceptedAttemptN || 1;\nif (!targetTaskId) {\n node.error('Missing target run_eval task id for judge_eval_attempt', msg);\n return null;\n}\nconst criteria = msg.evalJudgeCriteria || [];\nif (!Array.isArray(criteria) || criteria.length === 0) {\n node.error('Missing msg.evalJudgeCriteria', msg);\n return null;\n}\nmsg.evalJudgeTargetTaskId = targetTaskId;\nmsg.evalJudgeTargetAttemptN = targetAttemptN;\nmsg.payload = {\n taskType: 'judge_eval_attempt',\n title: 'Judge eval: ' + (msg.evalScenario?.id || 'unknown') + ' / ' + (msg.evalVariantLabel || 'unknown'),\n tags: ['eval-judge', msg.evalScenario?.id || 'unknown', msg.evalVariantLabel || 'unknown'],\n correlationId: msg.correlationId,\n input: {\n targetTaskId,\n targetAttemptN,\n successCriteria: {\n version: 1,\n rubric: {\n rubricId: msg.evalJudgeRubricId || 'ab-eval-rubric',\n version: msg.evalJudgeRubricVersion || '1',\n contentHash: msg.evalJudgeRubricHash || 'node-red-rubric',\n criteria\n }\n }\n }\n};\nreturn msg;",
|
|
112
|
+
"id": "sf_ab_build_judge_eval",
|
|
113
|
+
"name": "build judge_eval_attempt",
|
|
114
|
+
"outputs": 1,
|
|
115
|
+
"type": "function",
|
|
116
|
+
"wires": [["sf_ab_create_judge_eval"]],
|
|
117
|
+
"x": 460,
|
|
118
|
+
"y": 210,
|
|
119
|
+
"z": "subflow_ab_eval_with_judge"
|
|
120
|
+
},
|
|
121
|
+
{
|
|
122
|
+
"agent": "eval_agent_cfg",
|
|
123
|
+
"generateCorrelationId": false,
|
|
124
|
+
"id": "sf_ab_create_judge_eval",
|
|
125
|
+
"maxAttempts": 1,
|
|
126
|
+
"name": "JUDGE EVAL",
|
|
127
|
+
"runtimeProfile": "eval_profile_judge",
|
|
128
|
+
"type": "moltnet-tasks-create",
|
|
129
|
+
"wires": [["sf_ab_wait_judge_eval"]],
|
|
130
|
+
"x": 220,
|
|
131
|
+
"y": 290,
|
|
132
|
+
"z": "subflow_ab_eval_with_judge"
|
|
133
|
+
},
|
|
134
|
+
{
|
|
135
|
+
"agent": "eval_agent_cfg",
|
|
136
|
+
"id": "sf_ab_wait_judge_eval",
|
|
137
|
+
"kinds": "",
|
|
138
|
+
"name": "wait JUDGE EVAL",
|
|
139
|
+
"pollIntervalSec": 5,
|
|
140
|
+
"tail": true,
|
|
141
|
+
"taskId": "",
|
|
142
|
+
"timeoutSec": 1800,
|
|
143
|
+
"type": "moltnet-task-wait",
|
|
144
|
+
"wires": [[], ["sf_ab_stash_judge_eval"]],
|
|
145
|
+
"x": 430,
|
|
146
|
+
"y": 290,
|
|
147
|
+
"z": "subflow_ab_eval_with_judge"
|
|
148
|
+
},
|
|
149
|
+
{
|
|
150
|
+
"func": "msg.evalJudgeTaskId = msg.payload?.taskId || msg.payload?.task?.id || msg.payload?.id || msg.taskId || null;\nreturn msg;",
|
|
151
|
+
"id": "sf_ab_stash_judge_eval",
|
|
152
|
+
"name": "stash judge task id",
|
|
153
|
+
"outputs": 1,
|
|
154
|
+
"type": "function",
|
|
155
|
+
"wires": [["sf_ab_read_judge_eval"]],
|
|
156
|
+
"x": 660,
|
|
157
|
+
"y": 290,
|
|
158
|
+
"z": "subflow_ab_eval_with_judge"
|
|
159
|
+
},
|
|
160
|
+
{
|
|
161
|
+
"artifactKind": "",
|
|
162
|
+
"artifactTitle": "",
|
|
163
|
+
"id": "sf_ab_read_judge_eval",
|
|
164
|
+
"name": "read JUDGE EVAL",
|
|
165
|
+
"role": "context",
|
|
166
|
+
"source": "payload",
|
|
167
|
+
"type": "moltnet-task-reader",
|
|
168
|
+
"wires": [["sf_ab_store_delta"]],
|
|
169
|
+
"x": 880,
|
|
170
|
+
"y": 290,
|
|
171
|
+
"z": "subflow_ab_eval_with_judge"
|
|
172
|
+
},
|
|
173
|
+
{
|
|
174
|
+
"func": "function scoreFromComposite(judgment) {\n return typeof judgment.composite === 'number' ? Math.round(judgment.composite * 100) : null;\n}\n\nfunction buildVariantRecord(msg, judgment, judgeScore0to100) {\n return {\n correlationId: msg.correlationId || msg.evalGroupCorrelationId || null,\n scenarioId: msg.evalScenario?.id || 'unknown',\n variantLabel: msg.evalVariantLabel || judgment.variantLabel || 'unknown',\n runEvalTaskId: msg.evalJudgeTargetTaskId || msg.evalTaskId || null,\n runEvalAttemptN: msg.evalJudgeTargetAttemptN || msg.evalAcceptedAttemptN || null,\n judgeTaskId: msg.evalJudgeTaskId || null,\n producerScore0to100: msg.evalProducerScore?.score0to100 ?? null,\n judgeScore0to100,\n judgeComposite: typeof judgment.composite === 'number' ? judgment.composite : null,\n verdict: judgment.verdict || null,\n scores: judgment.scores || [],\n updatedAt: new Date().toISOString()\n };\n}\n\nfunction upsertScenarioResult(record) {\n const groups = flow.get('abEvalResults') || {};\n const group = groups[record.correlationId] || {\n correlationId: record.correlationId,\n scenarios: {},\n createdAt: new Date().toISOString()\n };\n const scenario = group.scenarios[record.scenarioId] || {};\n scenario[record.variantLabel] = record;\n group.scenarios[record.scenarioId] = scenario;\n group.updatedAt = new Date().toISOString();\n groups[record.correlationId] = group;\n flow.set('abEvalResults', groups);\n return scenario;\n}\n\nfunction rankedVariants(scenario) {\n return Object.values(scenario)\n .filter((item) => typeof item.judgeScore0to100 === 'number')\n .sort((a, b) => b.judgeScore0to100 - a.judgeScore0to100);\n}\n\nfunction buildDelta(correlationId, scenarioId, scenario) {\n const baseline = scenario['baseline-no-skill'];\n const skill = scenario['skill-rubric-v1'];\n if (!baseline || !skill) return null;\n return {\n correlationId,\n scenarioId,\n baselineScore0to100: baseline.judgeScore0to100,\n skillScore0to100: skill.judgeScore0to100,\n judgeDelta0to100: baseline.judgeScore0to100 === null || skill.judgeScore0to100 === null ? null : skill.judgeScore0to100 - baseline.judgeScore0to100,\n baselineProducerScore0to100: baseline.producerScore0to100,\n skillProducerScore0to100: skill.producerScore0to100,\n producerDelta0to100: baseline.producerScore0to100 === null || skill.producerScore0to100 === null ? null : skill.producerScore0to100 - baseline.producerScore0to100\n };\n}\n\nconst judgment = msg.payload || {};\nconst judgeScore0to100 = scoreFromComposite(judgment);\nconst record = buildVariantRecord(msg, judgment, judgeScore0to100);\nconst scenario = upsertScenarioResult(record);\nconst winner = rankedVariants(scenario)[0] || null;\n\nmsg.payload = {\n correlationId: record.correlationId,\n scenarioId: record.scenarioId,\n variantLabel: record.variantLabel,\n producerScore0to100: record.producerScore0to100,\n judgeScore0to100,\n verdict: judgment.verdict || null,\n scores: judgment.scores || [],\n winner,\n delta: buildDelta(record.correlationId, record.scenarioId, scenario),\n variants: scenario\n};\nreturn msg;",
|
|
175
|
+
"id": "sf_ab_store_delta",
|
|
176
|
+
"name": "store judgment + delta",
|
|
177
|
+
"outputs": 1,
|
|
178
|
+
"type": "function",
|
|
179
|
+
"wires": [[]],
|
|
180
|
+
"x": 1110,
|
|
181
|
+
"y": 290,
|
|
182
|
+
"z": "subflow_ab_eval_with_judge"
|
|
183
|
+
},
|
|
184
|
+
{
|
|
185
|
+
"disabled": false,
|
|
186
|
+
"id": "ab_eval_demo_tab",
|
|
187
|
+
"info": "Imports the A/B eval with judge subflow. Fill the moltnet-agent config, optionally set producer/judge runtime profile IDs, then run matching agent daemons before injecting the sample scenario.",
|
|
188
|
+
"label": "MoltNet A/B Eval (example)",
|
|
189
|
+
"type": "tab"
|
|
190
|
+
},
|
|
191
|
+
{
|
|
192
|
+
"apiUrl": "https://api.themolt.net",
|
|
193
|
+
"clientId": "",
|
|
194
|
+
"diaryId": "",
|
|
195
|
+
"id": "eval_agent_cfg",
|
|
196
|
+
"name": "eval-agent",
|
|
197
|
+
"teamId": "",
|
|
198
|
+
"type": "moltnet-agent"
|
|
199
|
+
},
|
|
200
|
+
{
|
|
201
|
+
"agent": "eval_agent_cfg",
|
|
202
|
+
"id": "eval_profile_producer",
|
|
203
|
+
"name": "eval-producer",
|
|
204
|
+
"profileId": "",
|
|
205
|
+
"profileName": "",
|
|
206
|
+
"type": "moltnet-runtime-profile"
|
|
207
|
+
},
|
|
208
|
+
{
|
|
209
|
+
"agent": "eval_agent_cfg",
|
|
210
|
+
"id": "eval_profile_judge",
|
|
211
|
+
"name": "eval-judge",
|
|
212
|
+
"profileId": "",
|
|
213
|
+
"profileName": "",
|
|
214
|
+
"type": "moltnet-runtime-profile"
|
|
215
|
+
},
|
|
216
|
+
{
|
|
217
|
+
"id": "ab_eval_note",
|
|
218
|
+
"info": "Run at least one agent daemon that can claim run_eval and judge_eval_attempt tasks. If you set runtime profile IDs above, run one daemon per profile. Leave profile IDs blank to let any eligible daemon claim both tasks.",
|
|
219
|
+
"name": "Daemon setup",
|
|
220
|
+
"type": "comment",
|
|
221
|
+
"wires": [],
|
|
222
|
+
"x": 160,
|
|
223
|
+
"y": 40,
|
|
224
|
+
"z": "ab_eval_demo_tab"
|
|
225
|
+
},
|
|
226
|
+
{
|
|
227
|
+
"func": "function uuid() {\n return '10000000-1000-4000-8000-100000000000'.replace(/[018]/g, (c) =>\n (Number(c) ^ Math.floor(Math.random() * 16) >> Number(c) / 4).toString(16)\n );\n}\nmsg.correlationId = uuid();\nmsg.evalVariantLabel = 'skill-rubric-v1';\nmsg.evalSkillContext = {\n slug: 'example-skill-context',\n binding: 'skill',\n content: 'Prefer concise structured analysis. Ground every recommendation in supplied evidence.'\n};\nmsg.evalScenario = {\n id: 'demo-readiness',\n title: 'Demo traffic-fit readiness scenario',\n evidence: {\n offer: 'Trial subscription for a niche productivity tool',\n landingPage: 'Mentions benefits but lacks pricing and proof',\n tracking: ['page_view'],\n constraints: ['No browsing', 'Use evidence only']\n },\n expected: {\n acceptableDecisions: ['needs_work', 'not_ready'],\n requiredFindings: ['pricing', 'proof', 'tracking'],\n forbidden: ['ready to scale']\n }\n};\nmsg.evalJudgeCriteria = [\n { id: 'json-validity', title: 'Valid structured output', weight: 0.2, description: 'Response is parseable and includes the required fields.' },\n { id: 'evidence-grounding', title: 'Evidence grounding', weight: 0.4, description: 'Findings use only supplied evidence and mark unknowns.' },\n { id: 'decision-quality', title: 'Decision quality', weight: 0.4, description: 'Decision and actions match the scenario expectations.' }\n];\nreturn msg;",
|
|
228
|
+
"id": "ab_eval_seed",
|
|
229
|
+
"name": "seed sample eval",
|
|
230
|
+
"outputs": 1,
|
|
231
|
+
"type": "function",
|
|
232
|
+
"wires": [["ab_eval_subflow"]],
|
|
233
|
+
"x": 170,
|
|
234
|
+
"y": 120,
|
|
235
|
+
"z": "ab_eval_demo_tab"
|
|
236
|
+
},
|
|
237
|
+
{
|
|
238
|
+
"crontab": "",
|
|
239
|
+
"id": "ab_eval_inject",
|
|
240
|
+
"name": "run sample",
|
|
241
|
+
"once": false,
|
|
242
|
+
"onceDelay": 0.1,
|
|
243
|
+
"payload": "",
|
|
244
|
+
"payloadType": "date",
|
|
245
|
+
"props": [
|
|
246
|
+
{
|
|
247
|
+
"p": "payload"
|
|
248
|
+
}
|
|
249
|
+
],
|
|
250
|
+
"repeat": "",
|
|
251
|
+
"topic": "",
|
|
252
|
+
"type": "inject",
|
|
253
|
+
"wires": [["ab_eval_seed"]],
|
|
254
|
+
"x": 110,
|
|
255
|
+
"y": 180,
|
|
256
|
+
"z": "ab_eval_demo_tab"
|
|
257
|
+
},
|
|
258
|
+
{
|
|
259
|
+
"id": "ab_eval_subflow",
|
|
260
|
+
"name": "A/B eval with judge",
|
|
261
|
+
"type": "subflow:subflow_ab_eval_with_judge",
|
|
262
|
+
"wires": [["ab_eval_debug"]],
|
|
263
|
+
"x": 410,
|
|
264
|
+
"y": 120,
|
|
265
|
+
"z": "ab_eval_demo_tab"
|
|
266
|
+
},
|
|
267
|
+
{
|
|
268
|
+
"active": true,
|
|
269
|
+
"complete": "payload",
|
|
270
|
+
"console": false,
|
|
271
|
+
"id": "ab_eval_debug",
|
|
272
|
+
"name": "eval result",
|
|
273
|
+
"statusType": "auto",
|
|
274
|
+
"statusVal": "",
|
|
275
|
+
"targetType": "msg",
|
|
276
|
+
"tosidebar": true,
|
|
277
|
+
"tostatus": false,
|
|
278
|
+
"type": "debug",
|
|
279
|
+
"wires": [],
|
|
280
|
+
"x": 650,
|
|
281
|
+
"y": 120,
|
|
282
|
+
"z": "ab_eval_demo_tab"
|
|
283
|
+
},
|
|
284
|
+
{
|
|
285
|
+
"env": [],
|
|
286
|
+
"id": "ab_eval_modules",
|
|
287
|
+
"modules": {
|
|
288
|
+
"@themoltnet/node-red-contrib-core": "0.2.0"
|
|
289
|
+
},
|
|
290
|
+
"type": "global-config"
|
|
291
|
+
}
|
|
292
|
+
]
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@themoltnet/node-red-contrib-core",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Node-RED nodes for the MoltNet API",
|
|
6
6
|
"keywords": [
|
|
@@ -24,11 +24,13 @@
|
|
|
24
24
|
"moltnet-agent": "dist/nodes/agent.js",
|
|
25
25
|
"moltnet-runtime-profile": "dist/nodes/runtime-profile.js",
|
|
26
26
|
"moltnet-tasks-create": "dist/nodes/tasks-create.js",
|
|
27
|
+
"moltnet-tasks-list": "dist/nodes/tasks-list.js",
|
|
27
28
|
"moltnet-task-get": "dist/nodes/task-get.js",
|
|
28
29
|
"moltnet-task-wait": "dist/nodes/task-wait.js",
|
|
29
30
|
"moltnet-workflow-status": "dist/nodes/workflow-status.js",
|
|
30
31
|
"moltnet-task-builder": "dist/nodes/task-builder.js",
|
|
31
|
-
"moltnet-task-reader": "dist/nodes/task-reader.js"
|
|
32
|
+
"moltnet-task-reader": "dist/nodes/task-reader.js",
|
|
33
|
+
"moltnet-entries-search": "dist/nodes/entries-search.js"
|
|
32
34
|
}
|
|
33
35
|
},
|
|
34
36
|
"engines": {
|