@pipefy/pipefy-process-coder 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/ppc-pipefy-flow-authoring/SKILL.md +262 -0
- package/.agents/skills/ppc-pipefy-flow-authoring/examples/01-main-2-danfe-consulta.json +247 -0
- package/.agents/skills/ppc-pipefy-flow-authoring/examples/01-main-2-webhook-retorno-consulta.json +589 -0
- package/.agents/skills/ppc-pipefy-flow-authoring/examples/01-main-4-recebimento-barramento.json +1391 -0
- package/.agents/skills/ppc-pipefy-flow-authoring/examples/02-subflow-2-danfe-retorno.json +623 -0
- package/.agents/skills/ppc-pipefy-flow-authoring/examples/02-subflow-4-criacao-operacao.json +636 -0
- package/.agents/skills/ppc-pipefy-flow-authoring/examples/03-subflow-4-criacao-titulo.json +3642 -0
- package/.agents/skills/ppc-pipefy-flow-authoring/examples/04-subflow-4-criacao-cedente.json +863 -0
- package/.agents/skills/ppc-pipefy-flow-authoring/examples/04-subflow-4-criacao-sacado.json +799 -0
- package/.agents/skills/ppc-pipefy-flow-authoring/examples/README.md +42 -0
- package/.agents/skills/ppc-pipefy-flow-authoring/examples/hml-01-subflow-1-titulo-acompanhamento-cobranca.json +581 -0
- package/.agents/skills/ppc-pipefy-flow-authoring/examples/hml-01-subflow-1-titulo-nfe-monitoramento.json +503 -0
- package/.agents/skills/ppc-pipefy-flow-authoring/examples/hml-01-subflow-1-titulo-retorno-bancario.json +562 -0
- package/.agents/skills/ppc-pipefy-flow-authoring/examples/hml-02-subflow-2-retorno-consulta-cedente.json +557 -0
- package/.agents/skills/ppc-pipefy-flow-authoring/examples/hml-02-subflow-2-retorno-consulta-sacado.json +609 -0
- package/.agents/skills/ppc-pipefy-pipe-authoring/SKILL.md +146 -0
- package/.agents/skills/ppc-pipefy-process-design/SKILL.md +127 -0
- package/.agents/skills/ppc-pipefy-workspace/SKILL.md +91 -0
- package/AGENTS.md +456 -0
- package/README.md +808 -0
- package/bin/pipe.js +13 -0
- package/package.json +35 -0
- package/src/apply/adopt.ts +150 -0
- package/src/apply/agentops.ts +71 -0
- package/src/apply/compile.ts +875 -0
- package/src/apply/execute.ts +336 -0
- package/src/apply/flowops.ts +399 -0
- package/src/apply/idmap.ts +241 -0
- package/src/apply/mutations.ts +955 -0
- package/src/apply/registry.ts +430 -0
- package/src/apply/types.ts +134 -0
- package/src/cli/args.ts +88 -0
- package/src/cli.ts +211 -0
- package/src/codec/flow.ts +199 -0
- package/src/codec/pack.ts +103 -0
- package/src/codec/roundtrip.ts +94 -0
- package/src/codec/unpack.ts +198 -0
- package/src/commands/agents.ts +184 -0
- package/src/commands/apply.ts +1144 -0
- package/src/commands/context.ts +119 -0
- package/src/commands/create.ts +79 -0
- package/src/commands/diff.ts +314 -0
- package/src/commands/flows.ts +414 -0
- package/src/commands/misc.ts +644 -0
- package/src/commands/plan.ts +331 -0
- package/src/commands/pull.ts +567 -0
- package/src/commands/runs.ts +83 -0
- package/src/commands/skills.ts +137 -0
- package/src/commands/verify.ts +253 -0
- package/src/config.ts +168 -0
- package/src/diff/agents.ts +122 -0
- package/src/diff/diff.ts +1130 -0
- package/src/diff/flow.ts +318 -0
- package/src/diff/html.ts +322 -0
- package/src/diff/render.ts +101 -0
- package/src/model/payload.ts +154 -0
- package/src/model/tree.ts +99 -0
- package/src/model/volatile.ts +55 -0
- package/src/pipefy/agents.ts +165 -0
- package/src/pipefy/automations.ts +219 -0
- package/src/pipefy/capability.ts +119 -0
- package/src/pipefy/client.ts +267 -0
- package/src/pipefy/discovery.ts +209 -0
- package/src/pipefy/internal.ts +380 -0
- package/src/pipefy/ipaas.ts +365 -0
- package/src/pipefy/reconstruct.ts +775 -0
- package/src/pipefy/reference.ts +251 -0
- package/src/pipefy/snapshot.ts +245 -0
- package/src/pipefy/toolkit.ts +200 -0
- package/src/pipefy/toolkit_bearer.py +137 -0
- package/src/report/integrations.ts +231 -0
- package/src/report/run.ts +475 -0
- package/src/util/fsx.ts +45 -0
- package/src/util/git.ts +32 -0
- package/src/util/json.ts +55 -0
- package/src/util/log.ts +76 -0
- package/src/util/pool.ts +48 -0
- package/src/util/slug.ts +26 -0
- package/src/util/tui.ts +335 -0
- package/src/validate/index.ts +123 -0
- package/src/validate/integrity.ts +387 -0
- package/src/validate/reference.ts +136 -0
- package/src/validate/schema.ts +328 -0
- package/src/workspace/agents.ts +290 -0
- package/src/workspace/docs.ts +407 -0
- package/src/workspace/flows.ts +191 -0
- package/src/workspace/layout.ts +165 -0
- package/src/workspace/lock.ts +148 -0
- package/src/workspace/read.ts +165 -0
- package/src/workspace/reference.ts +24 -0
- package/src/workspace/stamp.ts +301 -0
- package/src/workspace/write.ts +225 -0
|
@@ -0,0 +1,567 @@
|
|
|
1
|
+
import { resolve } from 'node:path';
|
|
2
|
+
import { bool, int, str, requirePositional, type Parsed } from '../cli/args.ts';
|
|
3
|
+
import { mkdirp, writeFile, writeJson } from '../util/fsx.ts';
|
|
4
|
+
import { bold, dim, green, info, ok, out, step, UserError, warn, yellow } from '../util/log.ts';
|
|
5
|
+
import { fullCoverage, type PayloadEnvelope } from '../model/payload.ts';
|
|
6
|
+
import { unpack } from '../codec/unpack.ts';
|
|
7
|
+
import { Progress } from '../util/tui.ts';
|
|
8
|
+
import { assertRoundTrip } from '../codec/roundtrip.ts';
|
|
9
|
+
import { discover, resolveRepo, type RepoRef } from '../pipefy/discovery.ts';
|
|
10
|
+
import { pullAndWriteAgents } from './agents.ts';
|
|
11
|
+
import { foldFreshBehaviors, foldBehaviorsIntoEnvelope } from '../workspace/agents.ts';
|
|
12
|
+
import { repoDir } from '../workspace/layout.ts';
|
|
13
|
+
import { emptyLock, upsertRepo, writeLock, writeWorkspace } from '../workspace/lock.ts';
|
|
14
|
+
import { writeTree } from '../workspace/write.ts';
|
|
15
|
+
import { writeDocs } from '../workspace/docs.ts';
|
|
16
|
+
import { readReference } from '../pipefy/reference.ts';
|
|
17
|
+
import { loadReference } from '../workspace/reference.ts';
|
|
18
|
+
import { baselineFile, paths } from '../workspace/layout.ts';
|
|
19
|
+
import { connect } from './context.ts';
|
|
20
|
+
import type { Tree } from '../model/tree.ts';
|
|
21
|
+
import { RunRecorder, invocation } from '../report/run.ts';
|
|
22
|
+
import { pullAndWriteFlows } from './flows.ts';
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* `pipe pull <pipeId>` — the read path.
|
|
26
|
+
*
|
|
27
|
+
* Snapshots when the organization has them, reconstruction over GraphQL and the
|
|
28
|
+
* internal API when it does not. Both produce the same payload shape, so
|
|
29
|
+
* everything downstream is identical; only `_meta.json` and the coverage map
|
|
30
|
+
* record which one happened.
|
|
31
|
+
*
|
|
32
|
+
* A pull always costs at least one snapshot when snapshots are used, because a
|
|
33
|
+
* snapshot of an unchanged pipe still creates a new row and a new S3 object
|
|
34
|
+
* (PLAN.md §1.1). `--reuse-snapshot` uses the newest complete one instead, which
|
|
35
|
+
* costs nothing.
|
|
36
|
+
*/
|
|
37
|
+
|
|
38
|
+
export const help = `pipe pull <pipeId> [options]
|
|
39
|
+
|
|
40
|
+
--out <dir> workspace directory (default: ./<pipe-name>-workspace)
|
|
41
|
+
--depth <n> how far to walk connected repos (default 1)
|
|
42
|
+
--org <id> organization id, for the internal automations query
|
|
43
|
+
--reuse-snapshot use the newest complete snapshot instead of creating one
|
|
44
|
+
--no-snapshot force GraphQL reconstruction even if snapshots exist
|
|
45
|
+
--no-internal skip the internal API (automations and jumps become unknown)
|
|
46
|
+
--label <text> label the snapshot this pull creates
|
|
47
|
+
--skip-roundtrip do not run the codec self-check (not recommended)
|
|
48
|
+
--no-flows skip the pipe's iPaaS flows (they are read by default)
|
|
49
|
+
--report write a run report into runs/ (payload copy included)
|
|
50
|
+
--from-file <path> build the workspace from an exported payload instead of the
|
|
51
|
+
network. Pairs with \`pipe export\`, needs no token, and is how
|
|
52
|
+
you work on a pipe you can only read once.`;
|
|
53
|
+
|
|
54
|
+
export const run = async (p: Parsed) => {
|
|
55
|
+
const fromFile = str(p, 'from-file');
|
|
56
|
+
if (fromFile) return pullFromFile(p, fromFile);
|
|
57
|
+
|
|
58
|
+
const repoIdArg = requirePositional(p, 0, 'a pipe or database id');
|
|
59
|
+
|
|
60
|
+
const depth = int(p, 'depth', 1);
|
|
61
|
+
const forceReconstruct = bool(p, 'no-snapshot');
|
|
62
|
+
const noInternal = bool(p, 'no-internal');
|
|
63
|
+
const reuseSnapshot = bool(p, 'reuse-snapshot');
|
|
64
|
+
const skipRoundTrip = bool(p, 'skip-roundtrip');
|
|
65
|
+
const label = str(p, 'label');
|
|
66
|
+
|
|
67
|
+
const startedAt = new Date();
|
|
68
|
+
const net = await connect(p);
|
|
69
|
+
const caps = await net.capabilities();
|
|
70
|
+
const useSnapshots = caps.snapshots && !forceReconstruct;
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Resolve the argument before anything else uses it.
|
|
74
|
+
*
|
|
75
|
+
* A database answers to two ids — the numeric internal one that connector
|
|
76
|
+
* fields carry, and the api id that the API returns — so both spellings have
|
|
77
|
+
* to work on the command line. Everything downstream keys on the numeric form
|
|
78
|
+
* (`discover`, the caches, `emptyLock`, `baselineFile`), so the raw argument is
|
|
79
|
+
* resolved once here and never threaded past this point. That is also what
|
|
80
|
+
* stops one database being pulled twice under two identities.
|
|
81
|
+
*/
|
|
82
|
+
const resolved = await resolveRepo(net.client, repoIdArg);
|
|
83
|
+
if (!resolved) {
|
|
84
|
+
throw new UserError(
|
|
85
|
+
`"${repoIdArg}" is not a repo this token can see`,
|
|
86
|
+
'pass a pipe id, a database id, or a database api id such as dzmxcuxz',
|
|
87
|
+
);
|
|
88
|
+
}
|
|
89
|
+
const rootRepoId = resolved.id;
|
|
90
|
+
|
|
91
|
+
info('');
|
|
92
|
+
info(
|
|
93
|
+
bold(
|
|
94
|
+
`pipe pull ${rootRepoId}` +
|
|
95
|
+
(resolved.kind === 'table' ? ` ${dim(`database ${resolved.apiId}`)}` : ''),
|
|
96
|
+
),
|
|
97
|
+
);
|
|
98
|
+
info(
|
|
99
|
+
dim(
|
|
100
|
+
`read path: ${useSnapshots ? (reuseSnapshot ? 'existing snapshots' : 'new snapshots') : 'GraphQL reconstruction'}` +
|
|
101
|
+
`${caps.snapshots ? '' : ' (this endpoint has no snapshots)'}` +
|
|
102
|
+
` · depth ${depth}${noInternal ? ' · internal API skipped' : ''}`,
|
|
103
|
+
),
|
|
104
|
+
);
|
|
105
|
+
info('');
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* One read of one repo, whichever path applies, cached for discovery reuse.
|
|
109
|
+
*
|
|
110
|
+
* Failures are cached too. Discovery reads a repo and then the write loop
|
|
111
|
+
* reads it again; without caching the failure, a repo that could not be read
|
|
112
|
+
* gets a *second* snapshot created for it — and a snapshot is a fresh S3
|
|
113
|
+
* object every time (PLAN.md §1.1). One failed pull should cost one snapshot,
|
|
114
|
+
* not two.
|
|
115
|
+
*/
|
|
116
|
+
const cache = new Map<number, PayloadEnvelope>();
|
|
117
|
+
const failed = new Set<number>();
|
|
118
|
+
const readRepo = async (ref: RepoRef): Promise<PayloadEnvelope | null> => {
|
|
119
|
+
const hit = cache.get(ref.id);
|
|
120
|
+
if (hit) return hit;
|
|
121
|
+
if (failed.has(ref.id)) return null;
|
|
122
|
+
|
|
123
|
+
try {
|
|
124
|
+
let env: PayloadEnvelope;
|
|
125
|
+
/**
|
|
126
|
+
* Databases take the snapshot path too.
|
|
127
|
+
*
|
|
128
|
+
* This used to refuse them, on the grounds that ask 10 — whether
|
|
129
|
+
* `createRepoSnapshot` accepts a table id — was untested and nobody wanted
|
|
130
|
+
* to find out on a client's data. It was tested on a five-record throwaway:
|
|
131
|
+
* it does, `SnapshotClient` needed no change because it reaches snapshots
|
|
132
|
+
* through `pipe(id:)`, and the payload that comes back has the same twenty
|
|
133
|
+
* relation arrays a pipe's does (VALIDATION.md §3g).
|
|
134
|
+
*/
|
|
135
|
+
if (useSnapshots) {
|
|
136
|
+
env = await readViaSnapshot(net, ref, { reuseSnapshot, label });
|
|
137
|
+
} else {
|
|
138
|
+
env = await net.reconstructor.reconstruct(ref.id, {
|
|
139
|
+
organizationId: net.organizationId,
|
|
140
|
+
noInternal,
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
cache.set(ref.id, env);
|
|
144
|
+
return env;
|
|
145
|
+
} catch (e) {
|
|
146
|
+
failed.add(ref.id);
|
|
147
|
+
warn(`could not read ${ref.kind} ${ref.id} (${ref.name ?? '?'}): ${(e as Error).message}`);
|
|
148
|
+
return null;
|
|
149
|
+
}
|
|
150
|
+
};
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* A pull is mostly waiting: ~63s per snapshot, plus an N+1 automation read on
|
|
154
|
+
* the reconstruction path. One task per repo, with the current stage as its
|
|
155
|
+
* detail, is what makes that legible rather than looking hung.
|
|
156
|
+
*/
|
|
157
|
+
const ui = new Progress({ title: `pulling repo ${rootRepoId}` });
|
|
158
|
+
ui.add({ id: 'discover', label: 'discovering connected repos', state: 'running' });
|
|
159
|
+
ui.start();
|
|
160
|
+
|
|
161
|
+
const repoTask = (ref: RepoRef) => {
|
|
162
|
+
const id = `repo:${ref.id}`;
|
|
163
|
+
if (!ui.has(id)) ui.add({ id, label: `${ref.kind} ${ref.id} ${ref.name ?? ''}`.trim(), depth: 1 });
|
|
164
|
+
return id;
|
|
165
|
+
};
|
|
166
|
+
|
|
167
|
+
let currentTask: string | null = null;
|
|
168
|
+
net.snapshots.onProgress = (text) => {
|
|
169
|
+
if (currentTask) ui.update(currentTask, { detail: text });
|
|
170
|
+
};
|
|
171
|
+
|
|
172
|
+
const discovery = await discover(net.client, rootRepoId, {
|
|
173
|
+
depth,
|
|
174
|
+
read: async (ref) => {
|
|
175
|
+
currentTask = repoTask(ref);
|
|
176
|
+
ui.update(currentTask, { state: 'running', detail: 'reading' });
|
|
177
|
+
return (await readRepo(ref))?.payload ?? null;
|
|
178
|
+
},
|
|
179
|
+
});
|
|
180
|
+
ui.update('discover', {
|
|
181
|
+
state: 'done',
|
|
182
|
+
detail: `${discovery.repos.length} repo${discovery.repos.length === 1 ? '' : 's'}`,
|
|
183
|
+
});
|
|
184
|
+
|
|
185
|
+
const rootRef = discovery.repos.find((r) => r.id === rootRepoId) as RepoRef;
|
|
186
|
+
const outDir = resolve(str(p, 'out') ?? `${slugName(rootRef.name ?? String(rootRepoId))}-workspace`);
|
|
187
|
+
await mkdirp(outDir);
|
|
188
|
+
|
|
189
|
+
const lock = emptyLock(rootRepoId, net.organizationId, depth);
|
|
190
|
+
lock.edges = discovery.edges;
|
|
191
|
+
|
|
192
|
+
const trees: Tree[] = [];
|
|
193
|
+
const sources = new Set<string>();
|
|
194
|
+
|
|
195
|
+
const flowSummaries: string[] = [];
|
|
196
|
+
const flowNotes: string[] = [];
|
|
197
|
+
const agentNotes: string[] = [];
|
|
198
|
+
|
|
199
|
+
for (const ref of discovery.repos) {
|
|
200
|
+
|
|
201
|
+
let env = await readRepo(ref);
|
|
202
|
+
if (!env) continue;
|
|
203
|
+
|
|
204
|
+
if (!skipRoundTrip) {
|
|
205
|
+
const rt = assertRoundTrip(env);
|
|
206
|
+
if (!rt.ok) {
|
|
207
|
+
warn(`codec round trip is not clean for ${ref.kind} ${ref.id}:`);
|
|
208
|
+
for (const d of rt.differences.slice(0, 8)) warn(` ${d.where}: ${d.detail}`);
|
|
209
|
+
warn('planning will refuse on this repo until the codec is fixed — see src/codec/');
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
const tree = unpack(env);
|
|
214
|
+
|
|
215
|
+
const task = repoTask(ref);
|
|
216
|
+
ui.update(task, { state: 'running', detail: 'writing files' });
|
|
217
|
+
|
|
218
|
+
const dir = repoDir(outDir, ref);
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* The repo's AI agents, read and written *before* the tree — agents.json
|
|
222
|
+
* files must exist and `automations/` has to know which rows are actually
|
|
223
|
+
* behaviors before `writeTree` decides where each one goes.
|
|
224
|
+
*
|
|
225
|
+
* Not in the payload at all (ROUTE-B-V2.md ask 5), so fetched separately and
|
|
226
|
+
* kept in `agents/`, mirroring `flows/`. Recording whether the read
|
|
227
|
+
* succeeded is what lets the validator distinguish "this pipe has no
|
|
228
|
+
* agents" from "I could not read them" — and only refuse an AI-referencing
|
|
229
|
+
* automation in the first case.
|
|
230
|
+
*/
|
|
231
|
+
if (ref.kind === 'pipe') {
|
|
232
|
+
ui.update(task, { detail: 'reading AI agents' });
|
|
233
|
+
try {
|
|
234
|
+
const agentsSection = await pullAndWriteAgents(net.client, ref.id, tree.repo['uuid'] as string | undefined, dir, {
|
|
235
|
+
workspaceRoot: outDir,
|
|
236
|
+
});
|
|
237
|
+
if (agentsSection.meta.coverage === 'complete') {
|
|
238
|
+
tree.automations = foldFreshBehaviors(tree.automations, agentsSection.agents);
|
|
239
|
+
/**
|
|
240
|
+
* The baseline gets the same replacement, or it is left holding the
|
|
241
|
+
* snapshot's stub forever — `pipe diff` would report a phantom
|
|
242
|
+
* change on every behavior on every pull, since the workspace files
|
|
243
|
+
* and the baseline could never agree.
|
|
244
|
+
*/
|
|
245
|
+
env = foldBehaviorsIntoEnvelope(env, agentsSection.agents);
|
|
246
|
+
} else {
|
|
247
|
+
agentNotes.push(`${ref.name ?? ref.id}: agents unknown — ${agentsSection.meta.reason ?? agentsSection.meta.notes.join('; ') ?? 'not readable'}`);
|
|
248
|
+
}
|
|
249
|
+
} catch (e) {
|
|
250
|
+
agentNotes.push(`${ref.name ?? ref.id}: agents could not be read — ${(e as Error).message}`);
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
trees.push(tree);
|
|
255
|
+
sources.add(env.source);
|
|
256
|
+
|
|
257
|
+
const written = await writeTree(dir, tree);
|
|
258
|
+
await writeJson(baselineFile(outDir, ref.id), env);
|
|
259
|
+
upsertRepo(lock, ref, outDir, tree);
|
|
260
|
+
|
|
261
|
+
/**
|
|
262
|
+
* The repo's iPaaS flows, read as part of the same pull.
|
|
263
|
+
*
|
|
264
|
+
* Not behind a flag because they are not an extra: a flow lives in exactly
|
|
265
|
+
* one pipe's iPaaS workspace, and this pipe's own webhook URLs carry the ids
|
|
266
|
+
* of its flows. Reading the automations without them gives a wrong picture of
|
|
267
|
+
* what the process does — iPaaS is where a process reaches other pipes and
|
|
268
|
+
* other systems, which the automation engine cannot.
|
|
269
|
+
*
|
|
270
|
+
* A pipe without iPaaS costs one refused token mint and records 'unknown'.
|
|
271
|
+
*/
|
|
272
|
+
if (!bool(p, 'no-flows') && ref.kind === 'pipe') {
|
|
273
|
+
ui.update(task, { detail: 'reading iPaaS flows' });
|
|
274
|
+
try {
|
|
275
|
+
const flowsSection = await pullAndWriteFlows(net.ipaas, ref.id, dir, { workspaceRoot: outDir });
|
|
276
|
+
if (flowsSection.meta.coverage === 'complete') {
|
|
277
|
+
flowSummaries.push(
|
|
278
|
+
`${ref.name ?? ref.id}: ${flowsSection.flows.length} flow(s), ${flowsSection.connections.length} connection(s)`,
|
|
279
|
+
);
|
|
280
|
+
for (const n of flowsSection.meta.notes) flowNotes.push(n);
|
|
281
|
+
} else {
|
|
282
|
+
flowNotes.push(`${ref.name ?? ref.id}: flows unknown — ${flowsSection.meta.reason ?? 'not readable'}`);
|
|
283
|
+
}
|
|
284
|
+
} catch (e) {
|
|
285
|
+
flowNotes.push(`${ref.name ?? ref.id}: flows could not be read — ${(e as Error).message}`);
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
const gaps = Object.entries(env.coverage).filter(([, c]) => c !== 'complete');
|
|
290
|
+
const fieldCount = tree.phases.reduce((n, ph) => n + ph.fields.length, 0);
|
|
291
|
+
/**
|
|
292
|
+
* A database is counted in its own terms.
|
|
293
|
+
*
|
|
294
|
+
* Its payload is a pipe's, so it does have phases — a start form plus one
|
|
295
|
+
* per record status. Reporting "3 phases" for a database with three columns
|
|
296
|
+
* and two statuses would be true and useless, so the statuses are named as
|
|
297
|
+
* what they are.
|
|
298
|
+
*/
|
|
299
|
+
const detail =
|
|
300
|
+
ref.kind === 'table'
|
|
301
|
+
? `${fieldCount} fields, ${Math.max(0, tree.phases.length - 1)} record statuses, ${written.length} files`
|
|
302
|
+
: `${tree.phases.length} phases, ${fieldCount} fields, ` +
|
|
303
|
+
`${tree.automations.length} automations, ${written.length} files`;
|
|
304
|
+
ui.update(task, {
|
|
305
|
+
state: 'done',
|
|
306
|
+
detail: detail + (gaps.length ? `, ${gaps.length} groups partial or unknown` : ''),
|
|
307
|
+
});
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
await writeLock(outDir, lock);
|
|
311
|
+
await writeWorkspace(outDir, {
|
|
312
|
+
name: rootRef.name ?? String(rootRepoId),
|
|
313
|
+
createdAt: new Date().toISOString(),
|
|
314
|
+
rootRepoId,
|
|
315
|
+
organizationId: net.organizationId,
|
|
316
|
+
depth,
|
|
317
|
+
lastPullSource: sources.size > 1 ? 'mixed' : ((sources.values().next().value ?? 'reconstructed') as 'snapshot' | 'reconstructed'),
|
|
318
|
+
tool: { name: 'pipefy-process-coder', version: '0.1.0' },
|
|
319
|
+
});
|
|
320
|
+
/**
|
|
321
|
+
* Read the closed sets while the client is open. Two queries, and it is what
|
|
322
|
+
* lets validate refuse a bad enum value or automation parameter offline with
|
|
323
|
+
* the value named, instead of the API answering `is invalid` partway through
|
|
324
|
+
* an apply.
|
|
325
|
+
*/
|
|
326
|
+
const reference = await readReference(net.client, rootRepoId);
|
|
327
|
+
if (reference) await writeJson(paths(outDir).reference, reference);
|
|
328
|
+
await writeDocs(outDir, lock, trees, caps, reference);
|
|
329
|
+
ui.stop();
|
|
330
|
+
|
|
331
|
+
/**
|
|
332
|
+
* A pull records itself only when asked. The workspace it just wrote already
|
|
333
|
+
* *is* the pipe, and the baseline envelope under `.ppc/baseline/` already is
|
|
334
|
+
* the payload, so a report here duplicates both. It earns its place when a
|
|
335
|
+
* sequence of runs is being kept side by side: this is the one that captures
|
|
336
|
+
* the pipe as it was before anyone edited anything.
|
|
337
|
+
*/
|
|
338
|
+
if (bool(p, 'report')) {
|
|
339
|
+
const rootEnv = cache.get(rootRepoId) ?? null;
|
|
340
|
+
const rec = new RunRecorder('pull', {
|
|
341
|
+
workspace: outDir,
|
|
342
|
+
repo: {
|
|
343
|
+
id: rootRepoId,
|
|
344
|
+
uuid: rootRef.uuid ?? null,
|
|
345
|
+
name: rootRef.name ?? String(rootRepoId),
|
|
346
|
+
kind: rootRef.kind,
|
|
347
|
+
},
|
|
348
|
+
command: invocation(),
|
|
349
|
+
startedAt,
|
|
350
|
+
});
|
|
351
|
+
rec.after(rootEnv);
|
|
352
|
+
rec.note(
|
|
353
|
+
`${discovery.repos.length} repo(s) discovered at depth ${depth}; ` +
|
|
354
|
+
`${trees.length} written (${lock.repos.filter((r) => r.kind === 'table').length} database(s))`,
|
|
355
|
+
);
|
|
356
|
+
for (const t of trees) for (const n of t.meta.notes) rec.note(n);
|
|
357
|
+
const res = await rec.write(
|
|
358
|
+
rootEnv ? 'ok' : 'failed',
|
|
359
|
+
rootEnv
|
|
360
|
+
? `pulled ${trees.length} repo(s) via ${[...sources].join(' + ')}`
|
|
361
|
+
: `could not read the root repo ${rootRepoId}`,
|
|
362
|
+
);
|
|
363
|
+
if (res?.error) warn(`could not write the run report: ${res.error}`);
|
|
364
|
+
else if (res) info(dim(`run recorded in runs/${rec.id}/`));
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
info('');
|
|
368
|
+
out(bold(outDir));
|
|
369
|
+
out(dim(' AGENTS.md · MAPPING.md · CAPABILITIES.md · REFERENCE.md written'));
|
|
370
|
+
|
|
371
|
+
if (flowSummaries.length) {
|
|
372
|
+
info('');
|
|
373
|
+
for (const s of flowSummaries) ok(`iPaaS ${s}`);
|
|
374
|
+
}
|
|
375
|
+
for (const n of flowNotes) info(dim(` ${n}`));
|
|
376
|
+
for (const n of agentNotes) info(dim(` ${n}`));
|
|
377
|
+
|
|
378
|
+
const databases = lock.repos.filter((r) => r.kind === 'table');
|
|
379
|
+
if (databases.length) {
|
|
380
|
+
info('');
|
|
381
|
+
ok(
|
|
382
|
+
`${databases.length} database(s): ` +
|
|
383
|
+
databases.map((r) => `${r.name} (${r.apiId})`).join(', '),
|
|
384
|
+
);
|
|
385
|
+
info(dim(' a database is a repo: its columns are fields on a start form, and the phases'));
|
|
386
|
+
info(dim(' after it are record statuses. Neither the statuses nor the records can be written.'));
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
const anyUnknown = lock.repos.flatMap((r) => Object.entries(r.coverage).filter(([, c]) => c === 'unknown').map(([e]) => e));
|
|
390
|
+
if (anyUnknown.length) {
|
|
391
|
+
const distinct = [...new Set(anyUnknown)].sort();
|
|
392
|
+
info('');
|
|
393
|
+
warn(`${distinct.length} entity group(s) are unknown, not empty: ${distinct.join(', ')}`);
|
|
394
|
+
info(dim(' They are never diffed and never applied. See AGENTS.md for what that means for edits.'));
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
const hasSecrets = trees.some((t) => t.webhooks.length > 0 || JSON.stringify(t.automations).includes('authentication'));
|
|
398
|
+
if (hasSecrets) {
|
|
399
|
+
info('');
|
|
400
|
+
warn('this workspace contains webhook URLs or HTTP-request credentials — possession is authorization');
|
|
401
|
+
info(dim(' a .gitignore was written; do not commit or share these files'));
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
info('');
|
|
405
|
+
info(`${yellow('next')} ${dim('cd')} ${outDir} ${dim('&&')} pipe validate`);
|
|
406
|
+
};
|
|
407
|
+
|
|
408
|
+
const readViaSnapshot = async (
|
|
409
|
+
net: Awaited<ReturnType<typeof connect>>,
|
|
410
|
+
ref: RepoRef,
|
|
411
|
+
opts: { reuseSnapshot: boolean; label?: string },
|
|
412
|
+
): Promise<PayloadEnvelope> => {
|
|
413
|
+
let versionId: string;
|
|
414
|
+
if (opts.reuseSnapshot) {
|
|
415
|
+
const latest = await net.snapshots.latestComplete(ref.id);
|
|
416
|
+
if (!latest) throw new Error('no complete snapshot exists to reuse — drop --reuse-snapshot');
|
|
417
|
+
versionId = latest.versionId;
|
|
418
|
+
step(`reusing snapshot ${versionId.slice(0, 8)} of ${ref.name ?? ref.id}`);
|
|
419
|
+
} else {
|
|
420
|
+
versionId = await net.snapshots.create(ref.id);
|
|
421
|
+
await net.snapshots.waitFor(ref.id, versionId);
|
|
422
|
+
if (opts.label) await net.snapshots.rename(ref.id, versionId, opts.label);
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
const { payload, gzip } = await net.snapshots.download(ref.id, versionId);
|
|
426
|
+
return {
|
|
427
|
+
source: 'snapshot',
|
|
428
|
+
versionId,
|
|
429
|
+
readAt: new Date().toISOString(),
|
|
430
|
+
repoKind: ref.kind,
|
|
431
|
+
coverage: fullCoverage('complete'),
|
|
432
|
+
notes: [],
|
|
433
|
+
payload,
|
|
434
|
+
...(gzip ? { _gzip: gzip } : {}),
|
|
435
|
+
} as PayloadEnvelope;
|
|
436
|
+
};
|
|
437
|
+
|
|
438
|
+
/*
|
|
439
|
+
* There was a `cacheSnapshotBytes` here, writing the raw .gz into
|
|
440
|
+
* `.ppc/snapshots/`. Nothing ever called it, so the directory was never created
|
|
441
|
+
* and the feature it implied — a local snapshot cache — did not exist.
|
|
442
|
+
*
|
|
443
|
+
* It is not missed. A re-pull already costs nothing with `--reuse-snapshot`,
|
|
444
|
+
* which reads the newest complete snapshot Pipefy already holds, and `pipe export`
|
|
445
|
+
* plus `pipe pull --from-file` covers working with no network at all. The baseline
|
|
446
|
+
* envelope keeps the payload itself, which is the part anything downstream reads.
|
|
447
|
+
*/
|
|
448
|
+
|
|
449
|
+
const slugName = (s: string) =>
|
|
450
|
+
s
|
|
451
|
+
.toLowerCase()
|
|
452
|
+
.replace(/[^a-z0-9]+/g, '-')
|
|
453
|
+
.replace(/^-+|-+$/g, '')
|
|
454
|
+
.slice(0, 40) || 'pipefy';
|
|
455
|
+
|
|
456
|
+
export const writeRawPayload = (path: string, env: PayloadEnvelope) => writeFile(path, JSON.stringify(env.payload, null, 2));
|
|
457
|
+
|
|
458
|
+
/**
|
|
459
|
+
* Build a workspace from an exported payload, with no network at all.
|
|
460
|
+
*
|
|
461
|
+
* Two reasons this is a first-class path rather than a test hook: a snapshot
|
|
462
|
+
* costs ~63s and a fresh S3 object every time (PLAN.md §1.1), and an FDE often
|
|
463
|
+
* gets one read of a client's pipe and then works offline. `pipe export` writes
|
|
464
|
+
* the file; this consumes it. Connected repos are recorded as edges but cannot
|
|
465
|
+
* be pulled, so they are reported rather than silently skipped.
|
|
466
|
+
*/
|
|
467
|
+
const pullFromFile = async (p: Parsed, fromFile: string) => {
|
|
468
|
+
const { readJson } = await import('../util/fsx.ts');
|
|
469
|
+
const { edgesFromPayload } = await import('../pipefy/discovery.ts');
|
|
470
|
+
const raw = await readJson<PayloadEnvelope | Record<string, unknown>>(resolve(fromFile));
|
|
471
|
+
|
|
472
|
+
const bare = !('payload' in raw && raw['payload']);
|
|
473
|
+
const env: PayloadEnvelope = bare
|
|
474
|
+
? {
|
|
475
|
+
/**
|
|
476
|
+
* A file with no envelope gets the only usable assumption — that every
|
|
477
|
+
* group was read in full — because the alternative, marking them all
|
|
478
|
+
* unknown, makes the file undiffable and unappliable.
|
|
479
|
+
*
|
|
480
|
+
* It is an assumption and it is announced below, because if the export
|
|
481
|
+
* came from a reconstruction it is wrong in the dangerous direction: a
|
|
482
|
+
* group nobody could read looks empty, and empty means delete.
|
|
483
|
+
*/
|
|
484
|
+
source: 'snapshot',
|
|
485
|
+
versionId: null,
|
|
486
|
+
readAt: new Date().toISOString(),
|
|
487
|
+
repoKind: 'pipe',
|
|
488
|
+
coverage: fullCoverage('complete'),
|
|
489
|
+
notes: [`read from ${fromFile}, not from the API`, 'no coverage envelope: every group assumed fully read'],
|
|
490
|
+
payload: raw as PayloadEnvelope['payload'],
|
|
491
|
+
}
|
|
492
|
+
: (raw as PayloadEnvelope);
|
|
493
|
+
|
|
494
|
+
const repoIdNum = Number(env.payload.id);
|
|
495
|
+
const rootRef: RepoRef = {
|
|
496
|
+
kind: env.repoKind,
|
|
497
|
+
id: repoIdNum,
|
|
498
|
+
apiId: String(repoIdNum),
|
|
499
|
+
name: String(env.payload['name'] ?? repoIdNum),
|
|
500
|
+
uuid: (env.payload['uuid'] as string | undefined) ?? null,
|
|
501
|
+
via: ['file'],
|
|
502
|
+
depth: 0,
|
|
503
|
+
};
|
|
504
|
+
|
|
505
|
+
const outDir = resolve(str(p, 'out') ?? `${slugName(rootRef.name ?? String(repoIdNum))}-workspace`);
|
|
506
|
+
await mkdirp(outDir);
|
|
507
|
+
|
|
508
|
+
info('');
|
|
509
|
+
info(bold(`pipe pull --from-file ${fromFile}`));
|
|
510
|
+
info(dim(`read path: file (${env.source} payload) · no network`));
|
|
511
|
+
info('');
|
|
512
|
+
|
|
513
|
+
if (bare) {
|
|
514
|
+
warn(`${fromFile} is a bare payload with no coverage envelope`);
|
|
515
|
+
info(dim(' every entity group is assumed to have been read in full, so an absent'));
|
|
516
|
+
info(dim(' webhook or automation will be treated as one to delete. If this came from'));
|
|
517
|
+
info(dim(' `pipe export` on an endpoint without snapshots, re-export with --envelope.'));
|
|
518
|
+
info('');
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
if (!bool(p, 'skip-roundtrip')) {
|
|
522
|
+
const rt = assertRoundTrip(env);
|
|
523
|
+
if (rt.ok) ok('codec round trip clean');
|
|
524
|
+
else {
|
|
525
|
+
warn('codec round trip is not clean for this payload:');
|
|
526
|
+
for (const d of rt.differences.slice(0, 8)) warn(` ${d.where}: ${d.detail}`);
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
const tree = unpack(env);
|
|
531
|
+
const lock = emptyLock(repoIdNum, str(p, 'org') ?? null, 0);
|
|
532
|
+
lock.edges = edgesFromPayload(env.payload).map((e) => ({ from: repoIdNum, ...e }));
|
|
533
|
+
|
|
534
|
+
const dir = repoDir(outDir, rootRef);
|
|
535
|
+
const written = await writeTree(dir, tree);
|
|
536
|
+
await writeJson(baselineFile(outDir, repoIdNum), env);
|
|
537
|
+
upsertRepo(lock, rootRef, outDir, tree);
|
|
538
|
+
await writeLock(outDir, lock);
|
|
539
|
+
await writeWorkspace(outDir, {
|
|
540
|
+
name: rootRef.name ?? String(repoIdNum),
|
|
541
|
+
createdAt: new Date().toISOString(),
|
|
542
|
+
rootRepoId: repoIdNum,
|
|
543
|
+
organizationId: str(p, 'org') ?? null,
|
|
544
|
+
depth: 0,
|
|
545
|
+
lastPullSource: env.source,
|
|
546
|
+
tool: { name: 'pipefy-process-coder', version: '0.1.0' },
|
|
547
|
+
});
|
|
548
|
+
await writeDocs(outDir, lock, [tree], null, await loadReference(outDir));
|
|
549
|
+
|
|
550
|
+
ok(
|
|
551
|
+
`${rootRef.kind} ${repoIdNum} ${green(rootRef.name ?? '')} — ${tree.phases.length} phases, ` +
|
|
552
|
+
`${tree.phases.reduce((n, ph) => n + ph.fields.length, 0)} fields, ${tree.automations.length} automations ` +
|
|
553
|
+
dim(`(${written.length} files)`),
|
|
554
|
+
);
|
|
555
|
+
|
|
556
|
+
const unreachable = [...new Set(lock.edges.map((e) => e.to))].filter((id) => id !== repoIdNum);
|
|
557
|
+
if (unreachable.length) {
|
|
558
|
+
info('');
|
|
559
|
+
warn(`this pipe connects to ${unreachable.length} other repo(s) that a file pull cannot fetch: ${unreachable.join(', ')}`);
|
|
560
|
+
info(dim(' export those too and pull them into their own workspaces, or pull from the API with --depth'));
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
info('');
|
|
564
|
+
out(bold(outDir));
|
|
565
|
+
info(`${yellow('next')} ${dim('cd')} ${outDir} ${dim('&&')} pipe validate`);
|
|
566
|
+
return 0;
|
|
567
|
+
};
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { join } from 'node:path';
|
|
2
|
+
import { promises as fs } from 'node:fs';
|
|
3
|
+
import { bool, str, type Parsed } from '../cli/args.ts';
|
|
4
|
+
import { bold, dim, green, info, out, red, UserError, yellow } from '../util/log.ts';
|
|
5
|
+
import { exists } from '../util/fsx.ts';
|
|
6
|
+
import { listRuns, runDir, type RunRecord } from '../report/run.ts';
|
|
7
|
+
import { resolveWorkspace } from './context.ts';
|
|
8
|
+
|
|
9
|
+
export const help = `pipe runs [workspace] [options]
|
|
10
|
+
|
|
11
|
+
--show <id> print one run's report
|
|
12
|
+
--last print the most recent run's report
|
|
13
|
+
--kind <kind> only pull / apply / dry-run / verify
|
|
14
|
+
--failed only runs that failed or partly applied
|
|
15
|
+
--json machine-readable
|
|
16
|
+
|
|
17
|
+
Every run folder holds report.md (readable, no credentials), run.json, and a copy
|
|
18
|
+
of the pipe and its integration surface as of the end of that run.`;
|
|
19
|
+
|
|
20
|
+
export const run = async (p: Parsed) => {
|
|
21
|
+
const root = await resolveWorkspace(p);
|
|
22
|
+
const runs = await listRuns(root);
|
|
23
|
+
|
|
24
|
+
const showId = str(p, 'show');
|
|
25
|
+
if (showId || bool(p, 'last')) {
|
|
26
|
+
const record = showId
|
|
27
|
+
? (runs.find((r) => r.id === showId) ?? runs.find((r) => r.id.includes(showId)))
|
|
28
|
+
: runs[runs.length - 1];
|
|
29
|
+
if (!record) {
|
|
30
|
+
throw new UserError(
|
|
31
|
+
showId ? `no run "${showId}" in ${root}` : `no runs recorded in ${root} yet`,
|
|
32
|
+
'run `pipe runs` to list what is there',
|
|
33
|
+
);
|
|
34
|
+
}
|
|
35
|
+
const path = join(runDir(root, record.id), 'report.md');
|
|
36
|
+
if (!(await exists(path))) throw new UserError(`${record.id} has no report.md`);
|
|
37
|
+
out(await fs.readFile(path, 'utf8'));
|
|
38
|
+
return 0;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
let filtered = runs;
|
|
42
|
+
const kind = str(p, 'kind');
|
|
43
|
+
if (kind) filtered = filtered.filter((r) => r.kind === kind);
|
|
44
|
+
if (bool(p, 'failed')) filtered = filtered.filter((r) => r.outcome === 'failed' || r.outcome === 'partial');
|
|
45
|
+
|
|
46
|
+
if (bool(p, 'json')) {
|
|
47
|
+
out(JSON.stringify(filtered, null, 2));
|
|
48
|
+
return 0;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
if (!filtered.length) {
|
|
52
|
+
info(runs.length ? 'no runs match that filter' : `no runs recorded in ${root} yet`);
|
|
53
|
+
return 0;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
out('');
|
|
57
|
+
out(bold(`${filtered.length} run${filtered.length === 1 ? '' : 's'} in ${root}`));
|
|
58
|
+
out('');
|
|
59
|
+
for (const r of filtered) {
|
|
60
|
+
out(` ${dim(r.startedAt.replace('T', ' ').slice(0, 19))} ${outcomeTag(r)} ${bold(r.kind.padEnd(8))} ${r.summary}`);
|
|
61
|
+
const parts = [
|
|
62
|
+
r.counts.steps ? `${r.counts.done}/${r.counts.steps} steps` : null,
|
|
63
|
+
r.counts.blocked ? red(`${r.counts.blocked} blocked`) : null,
|
|
64
|
+
r.verification ? (r.verification.ok ? 'verified' : red('verify failed')) : null,
|
|
65
|
+
`${(r.durationMs / 1000).toFixed(1)}s`,
|
|
66
|
+
].filter(Boolean);
|
|
67
|
+
out(` ${dim(r.id)} ${dim(parts.join(' · '))}`);
|
|
68
|
+
}
|
|
69
|
+
out('');
|
|
70
|
+
out(dim(` pipe runs --last the newest report in full`));
|
|
71
|
+
out(dim(` pipe runs --show <run id> any of them`));
|
|
72
|
+
out('');
|
|
73
|
+
return 0;
|
|
74
|
+
};
|
|
75
|
+
|
|
76
|
+
const outcomeTag = (r: RunRecord) =>
|
|
77
|
+
r.outcome === 'ok'
|
|
78
|
+
? green('ok ')
|
|
79
|
+
: r.outcome === 'partial'
|
|
80
|
+
? yellow('partial ')
|
|
81
|
+
: r.outcome === 'failed'
|
|
82
|
+
? red('failed ')
|
|
83
|
+
: dim(r.outcome.padEnd(8));
|