evals-lab 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +108 -0
- package/bin/run.js +36 -5
- package/lab/VERSION +1 -1
- package/lab/evals-core.mjs +464 -75
- package/lab/flows/flowApi.mjs +153 -0
- package/lab/flows/record.mjs +495 -0
- package/lab/metrics/builtin.mjs +113 -52
- package/lab/run-evals.js +32 -8
- package/lab/server.py +1174 -179
- package/lab/web/dist/assets/gallery-SnUhXRBn.js +3 -0
- package/lab/web/dist/assets/main-Ca7o-nM0.css +1 -0
- package/lab/web/dist/assets/main-Dru4_P5G.js +20 -0
- package/lab/web/dist/assets/tokens-0az9gfTq.js +58 -0
- package/lab/web/dist/assets/tokens-CqWJKhOx.css +1 -0
- package/lab/web/dist/gallery.html +3 -3
- package/lab/web/dist/index.html +4 -4
- package/package.json +1 -1
- package/lab/web/dist/assets/gallery-D_MxkfLj.js +0 -3
- package/lab/web/dist/assets/main-DDoeU6hq.css +0 -1
- package/lab/web/dist/assets/main-nz6Q4jVm.js +0 -21
- package/lab/web/dist/assets/tokens-Bl8cOqkf.js +0 -59
- package/lab/web/dist/assets/tokens-DCHW9ru7.css +0 -1
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
// The Power Automate management API, read from the browser
|
|
2
|
+
// (docs/power-automate.md § What the spike verified). This is the one place
|
|
3
|
+
// a Microsoft token is attached to a request, and it is attached to the Flow
|
|
4
|
+
// API's own address and nothing else: the lab's API client never carries
|
|
5
|
+
// one, so the server never sees one.
|
|
6
|
+
|
|
7
|
+
export const FLOW_API = "https://api.flow.microsoft.com/providers/Microsoft.ProcessSimple";
|
|
8
|
+
const VERSION = "api-version=2016-11-01";
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
/** One action's entry in a run, or one iteration of it: where its content is. */
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
// Run history's content sits behind pre-signed links on this domain, read
|
|
28
|
+
// with no token (docs/power-automate.md § What the spike verified).
|
|
29
|
+
const CONTENT_HOST = /\.powerplatformusercontent\.com$/;
|
|
30
|
+
/** How many content reads are in flight at once, and how often one is tried. */
|
|
31
|
+
const CONTENT_AT_ONCE = 4;
|
|
32
|
+
const CONTENT_TRIES = 4;
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
export function flowApi(token , fetchImpl = (u, i) => fetch(u, i)) {
|
|
37
|
+
const get = async (path ) => {
|
|
38
|
+
const url = `${FLOW_API}${path}${path.includes("?") ? "&" : "?"}${VERSION}`;
|
|
39
|
+
if (!url.startsWith(`${FLOW_API}/`)) throw new Error("the Flow API is the only address a Microsoft token goes to");
|
|
40
|
+
const r = await fetchImpl(url, { headers: { Authorization: `Bearer ${await token()}` } });
|
|
41
|
+
const text = await r.text();
|
|
42
|
+
let body = null;
|
|
43
|
+
try { body = JSON.parse(text); } catch { /* said below */ }
|
|
44
|
+
if (!r.ok) {
|
|
45
|
+
const code = (body )?.error?.code;
|
|
46
|
+
throw new Error(`Power Automate answered ${r.status}${code ? ` (${code})` : ""}`);
|
|
47
|
+
}
|
|
48
|
+
return body ;
|
|
49
|
+
};
|
|
50
|
+
const seg = encodeURIComponent;
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
const actionOf = (a ) => ({
|
|
56
|
+
status: a?.status ?? "",
|
|
57
|
+
startTime: a?.startTime ?? null,
|
|
58
|
+
error: a?.error ?? null,
|
|
59
|
+
inputsLink: a?.inputsLink?.uri ?? null,
|
|
60
|
+
outputsLink: a?.outputsLink?.uri ?? null,
|
|
61
|
+
at: Object.fromEntries((a?.repetitionIndexes ?? [])
|
|
62
|
+
.filter((r) => r.scopeName && typeof r.itemIndex === "number").map((r) => [r.scopeName , r.itemIndex ])),
|
|
63
|
+
});
|
|
64
|
+
// A next page's link, as a path `get` adds the version to again.
|
|
65
|
+
const pathOf = (link ) => {
|
|
66
|
+
const u = new URL(link);
|
|
67
|
+
u.searchParams.delete("api-version");
|
|
68
|
+
return `${u.pathname.slice(new URL(FLOW_API).pathname.length)}${u.search}`;
|
|
69
|
+
};
|
|
70
|
+
const runs = (env , flow ) => `/environments/${seg(env)}/flows/${seg(flow)}/runs`;
|
|
71
|
+
|
|
72
|
+
// Content reads share one queue, a few at a time, and a 429 waits as long
|
|
73
|
+
// as it is told to (or a little longer each try) before trying again.
|
|
74
|
+
let active = 0;
|
|
75
|
+
const waiting = [];
|
|
76
|
+
// A freed slot is handed to the next waiter rather than given back, so a
|
|
77
|
+
// read arriving between the two cannot take it as well.
|
|
78
|
+
const slot = async () => {
|
|
79
|
+
if (active < CONTENT_AT_ONCE) { active++; return; }
|
|
80
|
+
await new Promise ((go) => waiting.push(go));
|
|
81
|
+
};
|
|
82
|
+
const free = () => {
|
|
83
|
+
const next = waiting.shift();
|
|
84
|
+
if (next) next(); else active--;
|
|
85
|
+
};
|
|
86
|
+
const content = async (link ) => {
|
|
87
|
+
const u = new URL(link);
|
|
88
|
+
if (u.protocol !== "https:" || !CONTENT_HOST.test(u.hostname)) throw new Error("that is not run history's content");
|
|
89
|
+
await slot();
|
|
90
|
+
try {
|
|
91
|
+
for (let tried = 1; ; tried++) {
|
|
92
|
+
const r = await fetchImpl(link, {});
|
|
93
|
+
if (r.status === 429 && tried < CONTENT_TRIES) {
|
|
94
|
+
const after = Number(r.headers.get("Retry-After"));
|
|
95
|
+
await new Promise((go) => setTimeout(go, (after > 0 ? after : tried) * 1000));
|
|
96
|
+
continue;
|
|
97
|
+
}
|
|
98
|
+
if (!r.ok) throw new Error(`run history answered ${r.status}`);
|
|
99
|
+
const text = await r.text();
|
|
100
|
+
try { return JSON.parse(text); } catch { return text; }
|
|
101
|
+
}
|
|
102
|
+
} finally { free(); }
|
|
103
|
+
};
|
|
104
|
+
|
|
105
|
+
return {
|
|
106
|
+
environments: async () =>
|
|
107
|
+
((await get ("/environments")).value ?? [])
|
|
108
|
+
.map((e) => ({ id: e.name, name: e.properties?.displayName || e.name, isDefault: !!e.properties?.isDefault })),
|
|
109
|
+
flows: async (env ) =>
|
|
110
|
+
((await get (`/environments/${seg(env)}/flows`)).value ?? [])
|
|
111
|
+
.map((f) => ({ id: f.name, name: f.properties?.displayName || f.name, state: f.properties?.state ?? "" })),
|
|
112
|
+
flow: async (env , id ) => {
|
|
113
|
+
const f = await get (
|
|
114
|
+
`/environments/${seg(env)}/flows/${seg(id)}`);
|
|
115
|
+
return { id: f.name, name: f.properties?.displayName || f.name, definition: f.properties?.definition ?? {} };
|
|
116
|
+
},
|
|
117
|
+
/** A flow's runs, newest first, as far back as [limit] (the API pages them). */
|
|
118
|
+
runs: async (env , flow , limit = 250) => {
|
|
119
|
+
|
|
120
|
+
const out = [];
|
|
121
|
+
let path = runs(env, flow);
|
|
122
|
+
while (path && out.length < limit) {
|
|
123
|
+
const page = await get (path);
|
|
124
|
+
for (const r of page.value ?? []) {
|
|
125
|
+
out.push({ id: r.name, status: r.properties?.status ?? "", startTime: r.properties?.startTime ?? null,
|
|
126
|
+
error: r.properties?.error ?? null });
|
|
127
|
+
}
|
|
128
|
+
// The next page is the Flow API's own address, or it is not followed.
|
|
129
|
+
path = page.nextLink?.startsWith(`${FLOW_API}/`) ? pathOf(page.nextLink) : null;
|
|
130
|
+
}
|
|
131
|
+
return out.slice(0, limit);
|
|
132
|
+
},
|
|
133
|
+
/** A run, its trigger and each action's entry. */
|
|
134
|
+
run: async (env , flow , id ) => {
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
const r = await get (`${runs(env, flow)}/${seg(id)}?$expand=properties/actions`);
|
|
138
|
+
const p = r.properties ?? {};
|
|
139
|
+
return {
|
|
140
|
+
run: { id: r.name, status: p.status ?? "", startTime: p.startTime ?? null, error: p.error ?? null },
|
|
141
|
+
trigger: p.trigger ? actionOf(p.trigger) : null,
|
|
142
|
+
actions: Object.fromEntries(Object.entries(p.actions ?? {}).map(([n, a]) => [n, actionOf(a)])),
|
|
143
|
+
};
|
|
144
|
+
},
|
|
145
|
+
/** Every iteration of an action inside a loop, each with its own content. */
|
|
146
|
+
repetitions: async (env , flow , run , action ) =>
|
|
147
|
+
((await get (`${runs(env, flow)}/${seg(run)}/actions/${seg(action)}/repetitions`)).value ?? [])
|
|
148
|
+
.map((r) => actionOf(r.properties)),
|
|
149
|
+
/** What a content link holds: JSON parsed, anything else as text. */
|
|
150
|
+
content,
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
|
|
@@ -0,0 +1,495 @@
|
|
|
1
|
+
// A Power Automate workflow's records (docs/power-automate.md): one per call
|
|
2
|
+
// of a Power Automate step -- one per loop iteration -- holding what the
|
|
3
|
+
// step's request template read, the request production sent, and what came
|
|
4
|
+
// back. Read from run history, written by hand, or made from an uploaded
|
|
5
|
+
// file; the page builds them, and the server checks them again on the way in
|
|
6
|
+
// (server.py `record_problems`, this format's Python twin).
|
|
7
|
+
|
|
8
|
+
import { evaluate, } from "./wdl.mjs";
|
|
9
|
+
|
|
10
|
+
export const RECORD_FORMAT = "evals-lab/record";
|
|
11
|
+
export const RECORD_VERSION = 1;
|
|
12
|
+
|
|
13
|
+
/** What a step's template read, by where the expression reads it from. */
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
/** Production's result: the reply, or the step's failure. */
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
const isObj = (v ) => v != null && typeof v === "object" && !Array.isArray(v);
|
|
46
|
+
|
|
47
|
+
/** Every reason [value] is not a record this lab keeps, as sentences. */
|
|
48
|
+
export function recordProblems(value ) {
|
|
49
|
+
if (!isObj(value)) return ["a record is a JSON object"];
|
|
50
|
+
const bad = [];
|
|
51
|
+
if (value.format !== RECORD_FORMAT) bad.push(`a record says "format": "${RECORD_FORMAT}"`);
|
|
52
|
+
if (value.version !== RECORD_VERSION) bad.push(`this lab reads record version ${RECORD_VERSION}`);
|
|
53
|
+
if (typeof value.step !== "string" || !value.step) bad.push("a record names the step it is a call of");
|
|
54
|
+
if (value.at != null && typeof value.at !== "string") bad.push("a record's at is a time or null");
|
|
55
|
+
if (!isObj(value.scope)) bad.push("a record's scope is an object");
|
|
56
|
+
else {
|
|
57
|
+
for (const part of ["actions", "variables", "items"] ) {
|
|
58
|
+
if (value.scope[part] != null && !isObj(value.scope[part])) bad.push(`a record's scope.${part} is an object`);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
const req = value.request;
|
|
62
|
+
if (req != null && !(isObj(req) && typeof req.method === "string" && typeof req.uri === "string")) {
|
|
63
|
+
bad.push("a record's request is { method, uri, body } or null");
|
|
64
|
+
}
|
|
65
|
+
if (isObj(req) && "headers" in req) bad.push("a record keeps no request headers");
|
|
66
|
+
const res = value.result;
|
|
67
|
+
if (res != null && !(isObj(res) && (("status" in res && typeof res.status === "number") || isObj(res.error)))) {
|
|
68
|
+
bad.push("a record's result is { status, body }, { error } or null");
|
|
69
|
+
}
|
|
70
|
+
const origin = value.origin;
|
|
71
|
+
if (!isObj(origin) || !["run", "hand", "upload"].includes(origin.kind )) {
|
|
72
|
+
bad.push("a record's origin is run, hand or upload");
|
|
73
|
+
}
|
|
74
|
+
return bad;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
// ---- what a template reads -------------------------------------------------
|
|
78
|
+
|
|
79
|
+
/** One thing a step's template reads, and where a record keeps it. */
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
// A function call that reads flow data, in an expression anywhere in a
|
|
88
|
+
// template: the name, and its one quoted argument if it has one (`''` is a
|
|
89
|
+
// quote inside Workflow Definition Language's strings).
|
|
90
|
+
const READER = /\b(triggerBody|triggerOutputs|body|outputs|actions|variables|items|item|parameters)\(\s*(?:'((?:[^']|'')*)')?\s*\)/g;
|
|
91
|
+
|
|
92
|
+
/** Every expression in [template]: the whole of a string that begins `@`
|
|
93
|
+
(but not `@@`, a literal @), and each `@{…}` inside one. */
|
|
94
|
+
function expressions(template , out = []) {
|
|
95
|
+
if (typeof template === "string") {
|
|
96
|
+
if (template.startsWith("@") && !template.startsWith("@@") && !template.startsWith("@{")) out.push(template.slice(1));
|
|
97
|
+
for (const m of template.matchAll(/@\{([^}]*)\}/g)) out.push(m[1] );
|
|
98
|
+
} else if (Array.isArray(template)) {
|
|
99
|
+
for (const v of template) expressions(v, out);
|
|
100
|
+
} else if (isObj(template)) {
|
|
101
|
+
for (const v of Object.values(template)) expressions(v, out);
|
|
102
|
+
}
|
|
103
|
+
return out;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** What [template] reads, once each. [loop] is the loop it sits in, which
|
|
107
|
+
item() means. */
|
|
108
|
+
export function readsOf(template , loop = null) {
|
|
109
|
+
const seen = new Map ();
|
|
110
|
+
for (const expr of expressions(template)) {
|
|
111
|
+
for (const m of expr.matchAll(READER)) {
|
|
112
|
+
const fn = m[1] , arg = (m[2] ?? "").replace(/''/g, "'");
|
|
113
|
+
const read =
|
|
114
|
+
fn === "triggerBody" || fn === "triggerOutputs" ? { from: "trigger" }
|
|
115
|
+
: fn === "body" || fn === "outputs" || fn === "actions" ? (arg ? { from: "action", name: arg } : null)
|
|
116
|
+
: fn === "variables" ? (arg ? { from: "variable", name: arg } : null)
|
|
117
|
+
: fn === "items" ? (arg ? { from: "item", name: arg } : null)
|
|
118
|
+
: fn === "item" ? (loop ? { from: "item", name: loop } : null)
|
|
119
|
+
: arg ? { from: "parameter", name: arg } : null;
|
|
120
|
+
if (read) seen.set(JSON.stringify(read), read);
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
return [...seen.values()];
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** A record with an empty place for everything [reads] names: what a record
|
|
127
|
+
written by hand starts from. */
|
|
128
|
+
export function skeletonFor(step , reads ) {
|
|
129
|
+
const scope = {};
|
|
130
|
+
for (const r of reads) {
|
|
131
|
+
if (r.from === "trigger") scope.trigger = { body: null };
|
|
132
|
+
else if (r.from === "action") (scope.actions ??= {})[r.name] = { body: null };
|
|
133
|
+
else if (r.from === "variable") (scope.variables ??= {})[r.name] = null;
|
|
134
|
+
else if (r.from === "item") (scope.items ??= {})[r.name] = null;
|
|
135
|
+
}
|
|
136
|
+
return { format: RECORD_FORMAT, version: RECORD_VERSION, step, at: null, scope, request: null, result: null,
|
|
137
|
+
origin: { kind: "hand" } };
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/** What makes two records the same call. One from run history is its run's
|
|
141
|
+
call of the step at that iteration: imported twice, it is skipped the
|
|
142
|
+
second time, while another run's call -- the same request, sent again,
|
|
143
|
+
answered afresh -- is its own record, since a flow that sends the same
|
|
144
|
+
words every time is exactly where production's replies differ. One
|
|
145
|
+
written by hand or uploaded is the same as another that read and sent the
|
|
146
|
+
same things. */
|
|
147
|
+
export function recordKey(r ) {
|
|
148
|
+
if (r.origin?.kind === "run" && r.origin.run) {
|
|
149
|
+
return JSON.stringify({ step: r.step, run: r.origin.run, iteration: r.origin.iteration ?? "" });
|
|
150
|
+
}
|
|
151
|
+
const sorted = (v ) => Array.isArray(v) ? v.map(sorted)
|
|
152
|
+
: isObj(v) ? Object.fromEntries(Object.keys(v).sort().map(k => [k, sorted(v[k])])) : v;
|
|
153
|
+
return JSON.stringify(sorted({ step: r.step, scope: r.scope, request: r.request }));
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
// ---- a flow's Power Automate steps -------------------------------------------
|
|
157
|
+
|
|
158
|
+
/** A kind of action the lab can re-run as a Power Automate step: how its
|
|
159
|
+
request template is read off the definition. A registry, so a connector's
|
|
160
|
+
action is one more entry and no reader changes. */
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
export const FLOW_STEP_KINDS = Object.create(null);
|
|
169
|
+
|
|
170
|
+
// The built-in HTTP action: any endpoint (docs/power-automate.md).
|
|
171
|
+
FLOW_STEP_KINDS.Http = {
|
|
172
|
+
label: "HTTP",
|
|
173
|
+
template: ({ headers: _h, authentication: _a, ...rest }) => rest,
|
|
174
|
+
};
|
|
175
|
+
|
|
176
|
+
/** What a kind of flow action is to the lab as it walks a definition: a
|
|
177
|
+
loop (its inner actions run once per item, so a call inside one is a call
|
|
178
|
+
per iteration), or what declares a variable. A registry, so another
|
|
179
|
+
kind is one more entry and no walk changes. */
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
export const FLOW_ACTIONS = Object.create(null);
|
|
187
|
+
// Apply to each, and Do until: an inner action runs once per iteration.
|
|
188
|
+
FLOW_ACTIONS.Foreach = { loops: true };
|
|
189
|
+
FLOW_ACTIONS.Until = { loops: true };
|
|
190
|
+
// Initialize variable: `variables: [{ name, type, value }]`.
|
|
191
|
+
FLOW_ACTIONS.InitializeVariable = {
|
|
192
|
+
declares: (inputs) => (Array.isArray(inputs.variables) ? inputs.variables : [])
|
|
193
|
+
.flatMap((v ) => (isObj(v) && typeof v.name === "string" ? [v.name] : [])),
|
|
194
|
+
};
|
|
195
|
+
|
|
196
|
+
/** Whether an action of type [type] is a loop. */
|
|
197
|
+
const isLoop = (type ) => !!(type && FLOW_ACTIONS[type]?.loops);
|
|
198
|
+
|
|
199
|
+
/** A step of a flow the lab can re-run, as the definition has it. */
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
/** Every Power Automate step in [definition], however deep in scopes,
|
|
214
|
+
branches and loops. */
|
|
215
|
+
export function stepsOf(definition ) {
|
|
216
|
+
const out = [];
|
|
217
|
+
const walk = (actions , loops ) => {
|
|
218
|
+
for (const [name, a] of Object.entries(actions ?? {})) {
|
|
219
|
+
const kind = a.type ? FLOW_STEP_KINDS[a.type] : undefined;
|
|
220
|
+
if (kind && isObj(a.inputs)) {
|
|
221
|
+
const template = kind.template(a.inputs);
|
|
222
|
+
out.push({ name, kind: a.type , loops, template, reads: readsOf(template, loops.at(-1) ?? null) });
|
|
223
|
+
}
|
|
224
|
+
const inner = isLoop(a.type) ? [...loops, name] : loops;
|
|
225
|
+
walk(a.actions, inner);
|
|
226
|
+
walk(a.else?.actions, inner);
|
|
227
|
+
for (const branch of Object.values(a.cases ?? {})) walk(branch.actions, inner);
|
|
228
|
+
walk(a.default?.actions, inner);
|
|
229
|
+
}
|
|
230
|
+
};
|
|
231
|
+
walk(isObj(definition) ? (definition.actions ) : undefined, []);
|
|
232
|
+
return out;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/** Every action in [definition] by name, however deep: where a record's
|
|
236
|
+
import finds what declares a variable or a loop's items. */
|
|
237
|
+
export function actionsByName(definition ) {
|
|
238
|
+
const out = {};
|
|
239
|
+
const walk = (actions , loops ) => {
|
|
240
|
+
for (const [name, a] of Object.entries(actions ?? {})) {
|
|
241
|
+
out[name] = { ...a, loops };
|
|
242
|
+
const inner = isLoop(a.type) ? [...loops, name] : loops;
|
|
243
|
+
walk(a.actions, inner);
|
|
244
|
+
walk(a.else?.actions, inner);
|
|
245
|
+
for (const branch of Object.values(a.cases ?? {})) walk(branch.actions, inner);
|
|
246
|
+
walk(a.default?.actions, inner);
|
|
247
|
+
}
|
|
248
|
+
};
|
|
249
|
+
walk(isObj(definition) ? (definition.actions ) : undefined, []);
|
|
250
|
+
return out;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
// ---- a record from run history -------------------------------------------------
|
|
254
|
+
|
|
255
|
+
/** Where one call of a step sits in a run: each enclosing loop's item index. */
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
/** What an import reads a run's history through. The page answers it from
|
|
259
|
+
the Flow API (web/src/flows/history.ts); a check answers it from fixtures.
|
|
260
|
+
Each returns the content as run history stores it, or null for none. */
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
/** One call of [step], as run history holds it: its inputs and outputs, or
|
|
271
|
+
its failure, and when it ran. */
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
/** The record of [call]: everything [step]'s template read, fetched through
|
|
282
|
+
[history], and production's request and result. Nothing is redacted here --
|
|
283
|
+
the caller redacts the record before it leaves the browser. */
|
|
284
|
+
export async function recordFromCall(step , call , history ,
|
|
285
|
+
definition ) {
|
|
286
|
+
const actions = actionsByName(definition);
|
|
287
|
+
const scope = {};
|
|
288
|
+
for (const read of step.reads) {
|
|
289
|
+
if (read.from === "trigger") {
|
|
290
|
+
const out = await history.triggerOutputs();
|
|
291
|
+
scope.trigger = { body: isObj(out) ? out.body : out };
|
|
292
|
+
} else if (read.from === "action") {
|
|
293
|
+
const out = await history.actionOutputs(read.name, call.at);
|
|
294
|
+
(scope.actions ??= {})[read.name] = isObj(out)
|
|
295
|
+
? { body: out.body, ...(typeof out.statusCode === "number" ? { statusCode: out.statusCode } : {}) }
|
|
296
|
+
: { body: out };
|
|
297
|
+
} else if (read.from === "variable") {
|
|
298
|
+
// A variable is what the action that declares it set it to. A later
|
|
299
|
+
// Set or Append is not replayed: the record holds the declared value
|
|
300
|
+
// (docs/power-automate.md § Records).
|
|
301
|
+
const declares = (a ) => {
|
|
302
|
+
const kind = a.type ? FLOW_ACTIONS[a.type] : undefined;
|
|
303
|
+
return !!kind?.declares && isObj(a.inputs) && kind.declares(a.inputs).includes(read.name);
|
|
304
|
+
};
|
|
305
|
+
const decl = Object.entries(actions).find(([, a]) => declares(a));
|
|
306
|
+
const inputs = decl ? await history.actionInputs(decl[0], call.at) : null;
|
|
307
|
+
const vars = isObj(inputs) && Array.isArray(inputs.variables) ? inputs.variables : [];
|
|
308
|
+
const hit = vars.find((v ) => isObj(v) && v.name === read.name) ;
|
|
309
|
+
(scope.variables ??= {})[read.name] = hit ? hit.value : null;
|
|
310
|
+
} else if (read.from === "item") {
|
|
311
|
+
// A loop's current item: its foreachItems at this call's index.
|
|
312
|
+
const inputs = await history.actionInputs(read.name, call.at);
|
|
313
|
+
const items = isObj(inputs) && Array.isArray(inputs.foreachItems) ? inputs.foreachItems : [];
|
|
314
|
+
(scope.items ??= {})[read.name] = items[call.at[read.name] ?? 0] ?? null;
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
const req = isObj(call.inputs) ? call.inputs : null;
|
|
318
|
+
const out = isObj(call.outputs) ? call.outputs : null;
|
|
319
|
+
return {
|
|
320
|
+
format: RECORD_FORMAT, version: RECORD_VERSION, step: step.name, at: call.startTime, scope,
|
|
321
|
+
request: req ? { method: String(req.method ?? ""), uri: String(req.uri ?? ""), body: req.body ?? null } : null,
|
|
322
|
+
result: call.error && !out ? { error: call.error }
|
|
323
|
+
: out ? { status: typeof out.statusCode === "number" ? out.statusCode : 0, body: out.body ?? null } : null,
|
|
324
|
+
origin: { kind: "run", run: call.run,
|
|
325
|
+
...(Object.keys(call.at).length ? { iteration: Object.entries(call.at).map(([k, v]) => `${k}[${v}]`).join(",") } : {}) },
|
|
326
|
+
};
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
/** A record's file name: the step, then the run and iteration it came from. */
|
|
330
|
+
export function recordName(r ) {
|
|
331
|
+
const tail = r.origin.kind === "run"
|
|
332
|
+
? `${(r.origin.run ?? "").slice(-8)}${r.origin.iteration ? `-${r.origin.iteration.replace(/.*\[(\d+)\]$/, "$1")}` : ""}`
|
|
333
|
+
: new Date().toISOString().replace(/\D/g, "").slice(0, 14);
|
|
334
|
+
return `${r.step}-${tail}.json`.replace(/[^A-Za-z0-9._-]/g, "_");
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
// ---- records from an uploaded file ---------------------------------------------
|
|
338
|
+
|
|
339
|
+
/** The column a record's scope takes [read] from, when a file's rows are
|
|
340
|
+
made into records: the action's, variable's or loop's own name, or
|
|
341
|
+
`trigger` for the trigger's body (`trigger.x` for one field of it). */
|
|
342
|
+
export const columnOf = (read ) => read.from === "trigger" ? "trigger"
|
|
343
|
+
: read.from === "parameter" ? "" : read.name;
|
|
344
|
+
|
|
345
|
+
/** A cell's value: JSON when it reads as JSON, its text otherwise. */
|
|
346
|
+
const cellValue = (v ) => {
|
|
347
|
+
if (typeof v !== "string") return v;
|
|
348
|
+
try { return JSON.parse(v); } catch { return v; }
|
|
349
|
+
};
|
|
350
|
+
|
|
351
|
+
/** A record per row of [rows] for [step], each column matched by name to
|
|
352
|
+
what the step reads. The columns nothing reads are named, so the page can
|
|
353
|
+
say so before anything is kept. */
|
|
354
|
+
export function recordsFromRows(step , rows )
|
|
355
|
+
{
|
|
356
|
+
const columns = [...new Set(rows.flatMap(r => Object.keys(r)))];
|
|
357
|
+
const wanted = new Set(step.reads.map(columnOf).filter(Boolean));
|
|
358
|
+
const unmatched = columns.filter(c => !wanted.has(c) && !(wanted.has("trigger") && c.startsWith("trigger.")));
|
|
359
|
+
const records = rows.map(row => {
|
|
360
|
+
const r = skeletonFor(step.name, step.reads);
|
|
361
|
+
r.origin = { kind: "upload" };
|
|
362
|
+
for (const read of step.reads) {
|
|
363
|
+
const col = columnOf(read);
|
|
364
|
+
if (read.from === "trigger") {
|
|
365
|
+
const fields = Object.keys(row).filter(c => c.startsWith("trigger."));
|
|
366
|
+
const body = col in row ? cellValue(row[col])
|
|
367
|
+
: fields.length ? Object.fromEntries(fields.map(c => [c.slice("trigger.".length), cellValue(row[c])])) : null;
|
|
368
|
+
r.scope.trigger = { body };
|
|
369
|
+
} else if (col in row) {
|
|
370
|
+
const v = cellValue(row[col]);
|
|
371
|
+
if (read.from === "action") r.scope.actions [read.name] = { body: v };
|
|
372
|
+
else if (read.from === "variable") r.scope.variables [read.name] = v;
|
|
373
|
+
else if (read.from === "item") r.scope.items [read.name] = v;
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
return r;
|
|
377
|
+
});
|
|
378
|
+
return { records, unmatched };
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
/** A CSV file's rows, by its header line (RFC 4180: quoted fields, `""`
|
|
382
|
+
inside them, a line break inside quotes). */
|
|
383
|
+
export function csvRows(text ) {
|
|
384
|
+
const lines = [];
|
|
385
|
+
let row = [], cell = "", quoted = false;
|
|
386
|
+
const src = text.replace(/^/, "");
|
|
387
|
+
for (let i = 0; i < src.length; i++) {
|
|
388
|
+
const c = src[i] ;
|
|
389
|
+
if (quoted) {
|
|
390
|
+
if (c === '"' && src[i + 1] === '"') { cell += '"'; i++; }
|
|
391
|
+
else if (c === '"') quoted = false;
|
|
392
|
+
else cell += c;
|
|
393
|
+
} else if (c === '"') quoted = true;
|
|
394
|
+
else if (c === ",") { row.push(cell); cell = ""; }
|
|
395
|
+
else if (c === "\n" || c === "\r") {
|
|
396
|
+
if (c === "\r" && src[i + 1] === "\n") i++;
|
|
397
|
+
row.push(cell); lines.push(row); row = []; cell = "";
|
|
398
|
+
} else cell += c;
|
|
399
|
+
}
|
|
400
|
+
if (cell || row.length) { row.push(cell); lines.push(row); }
|
|
401
|
+
const [head, ...body] = lines.filter(l => l.some(v => v !== ""));
|
|
402
|
+
if (!head) return [];
|
|
403
|
+
return body.map(l => Object.fromEntries(head.map((h, i) => [h.trim(), l[i] ?? ""])));
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
// ---- fidelity: the template against production -----------------------------------
|
|
407
|
+
|
|
408
|
+
/** Whether [step]'s own template, evaluated against [record]'s scope, is the
|
|
409
|
+
request production sent: the proof the lab rebuilds a flow's request as
|
|
410
|
+
Power Automate does, before any wording is changed. `where` names the
|
|
411
|
+
first place they part, or why the template could not be evaluated. */
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
export function fidelityOf(step , record , definition ) {
|
|
415
|
+
if (!record.request) return { match: null };
|
|
416
|
+
const params = isObj(definition) && isObj(definition.parameters)
|
|
417
|
+
? Object.fromEntries(Object.entries(definition.parameters).map(([k, v]) => [k, isObj(v) ? v.defaultValue : undefined]))
|
|
418
|
+
: {};
|
|
419
|
+
let rebuilt ;
|
|
420
|
+
try {
|
|
421
|
+
const ctx = { scope: record.scope , loop: step.loops.at(-1) ?? null, now: record.at, parameters: params };
|
|
422
|
+
const t = step.template;
|
|
423
|
+
rebuilt = { method: evaluate(t.method, ctx), uri: evaluate(t.uri, ctx), body: evaluate(t.body ?? null, ctx) };
|
|
424
|
+
} catch (e) {
|
|
425
|
+
return { match: false, where: e instanceof Error ? e.message : String(e) };
|
|
426
|
+
}
|
|
427
|
+
const where = firstDifference(rebuilt, record.request, "request");
|
|
428
|
+
return where ? { match: false, where } : { match: true };
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
/** The first path at which [a] and [b] differ, or null: keys compared in
|
|
432
|
+
order, since a body is sent as written. */
|
|
433
|
+
function firstDifference(a , b , at ) {
|
|
434
|
+
if (Array.isArray(a) && Array.isArray(b)) {
|
|
435
|
+
if (a.length !== b.length) return `${at} has ${a.length} entries, production ${b.length}`;
|
|
436
|
+
for (let i = 0; i < a.length; i++) { const d = firstDifference(a[i], b[i], `${at}[${i}]`); if (d) return d; }
|
|
437
|
+
return null;
|
|
438
|
+
}
|
|
439
|
+
if (isObj(a) && isObj(b)) {
|
|
440
|
+
const ka = Object.keys(a), kb = Object.keys(b);
|
|
441
|
+
if (ka.join() !== kb.join()) return `${at} has keys ${ka.join(", ")}, production ${kb.join(", ")}`;
|
|
442
|
+
for (const k of ka) { const d = firstDifference(a[k], b[k], `${at}.${k}`); if (d) return d; }
|
|
443
|
+
return null;
|
|
444
|
+
}
|
|
445
|
+
return JSON.stringify(a) === JSON.stringify(b) ? null : `${at}`;
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
// ---- what the flow does with a step's reply -------------------------------------
|
|
449
|
+
|
|
450
|
+
/** A kind of action that reads a Power Automate step's reply before the flow
|
|
451
|
+
acts on it: the expression it reads it through. A registry, so another
|
|
452
|
+
reader (a Compose, a connector) is one more entry. */
|
|
453
|
+
|
|
454
|
+
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
|
|
460
|
+
export const FLOW_READERS = Object.create(null);
|
|
461
|
+
|
|
462
|
+
// Parse JSON: its content is the expression that turns the reply into what
|
|
463
|
+
// its schema checks -- the fence-stripping `trim(replace(…))`, say.
|
|
464
|
+
FLOW_READERS.ParseJson = {
|
|
465
|
+
reads: (inputs) => (typeof inputs.content === "string" ? inputs.content : null),
|
|
466
|
+
schema: (inputs) => inputs.schema ?? null,
|
|
467
|
+
};
|
|
468
|
+
|
|
469
|
+
/** The reader action that reads [step]'s reply: its name, the expression it
|
|
470
|
+
reads through, and its schema -- or null when none does. */
|
|
471
|
+
export function readerFor(definition , step ) {
|
|
472
|
+
for (const [name, a] of Object.entries(actionsByName(definition))) {
|
|
473
|
+
const kind = a.type ? FLOW_READERS[a.type] : undefined;
|
|
474
|
+
const expr = kind && isObj(a.inputs) ? kind.reads(a.inputs) : null;
|
|
475
|
+
if (expr && readsOf(expr).some((r) => r.from === "action" && r.name === step)) {
|
|
476
|
+
return { name, expression: expr, schema: kind .schema && isObj(a.inputs) ? kind .schema(a.inputs) ?? null : null };
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
return null;
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
/** How the flow reads [step]'s reply: the expression of the first reader
|
|
483
|
+
action that reads it, or null when none does. */
|
|
484
|
+
export const readerOf = (definition , step ) => readerFor(definition, step)?.expression ?? null;
|
|
485
|
+
|
|
486
|
+
/** The fields of [reader]'s output the rest of the flow reads --
|
|
487
|
+
`body('Parse_JSON')['category']` in a Condition -- in the order they appear:
|
|
488
|
+
what production's decisions turn on, so what a new wording has to keep. */
|
|
489
|
+
export function fieldsRead(definition , reader ) {
|
|
490
|
+
const text = JSON.stringify(isObj(definition) ? definition.actions ?? {} : {});
|
|
491
|
+
const name = reader.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/'/g, "''");
|
|
492
|
+
const found = new Set ();
|
|
493
|
+
for (const m of text.matchAll(new RegExp(`body\\('${name}'\\)\\??\\['((?:[^']|'')*)'\\]`, "g"))) found.add(m[1] .replace(/''/g, "'"));
|
|
494
|
+
return [...found];
|
|
495
|
+
}
|