@shipfox/api-workflows 12.1.0 → 12.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +4 -4
- package/CHANGELOG.md +29 -0
- package/dist/core/entities/job.d.ts +1 -1
- package/dist/core/entities/job.d.ts.map +1 -1
- package/dist/core/entities/step.d.ts +2 -2
- package/dist/core/entities/step.d.ts.map +1 -1
- package/dist/core/errors.d.ts +16 -0
- package/dist/core/errors.d.ts.map +1 -1
- package/dist/core/errors.js +18 -0
- package/dist/core/errors.js.map +1 -1
- package/dist/core/step-config/assemble-run-context.d.ts +15 -5
- package/dist/core/step-config/assemble-run-context.d.ts.map +1 -1
- package/dist/core/step-config/assemble-run-context.js +77 -27
- package/dist/core/step-config/assemble-run-context.js.map +1 -1
- package/dist/core/step-config/fields.d.ts +5 -0
- package/dist/core/step-config/fields.d.ts.map +1 -1
- package/dist/core/step-config/fields.js +21 -3
- package/dist/core/step-config/fields.js.map +1 -1
- package/dist/core/step-config/index.d.ts +1 -1
- package/dist/core/step-config/index.d.ts.map +1 -1
- package/dist/core/step-config/index.js +1 -1
- package/dist/core/step-config/index.js.map +1 -1
- package/dist/core/step-config/job-output-limits.d.ts +11 -0
- package/dist/core/step-config/job-output-limits.d.ts.map +1 -0
- package/dist/core/step-config/job-output-limits.js +88 -0
- package/dist/core/step-config/job-output-limits.js.map +1 -0
- package/dist/core/step-config/materialize-workflow-model.d.ts +1 -1
- package/dist/core/step-config/materialize-workflow-model.d.ts.map +1 -1
- package/dist/core/step-config/materialize-workflow-model.js +41 -14
- package/dist/core/step-config/materialize-workflow-model.js.map +1 -1
- package/dist/db/db.d.ts +50 -50
- package/dist/db/job-listeners.d.ts.map +1 -1
- package/dist/db/job-listeners.js +6 -5
- package/dist/db/job-listeners.js.map +1 -1
- package/dist/db/schema/job-executions.d.ts +3 -3
- package/dist/db/schema/job-listener-events.d.ts +1 -1
- package/dist/db/schema/jobs.d.ts +7 -7
- package/dist/db/schema/outbox.d.ts +1 -1
- package/dist/db/schema/step-attempts.d.ts +2 -2
- package/dist/db/schema/steps.d.ts +5 -5
- package/dist/db/schema/workflow-run-attempts.d.ts +3 -3
- package/dist/db/schema/workflow-run-counters.d.ts +1 -1
- package/dist/db/schema/workflow-runs.d.ts +2 -2
- package/dist/db/workflow-runs/job-executions.d.ts.map +1 -1
- package/dist/db/workflow-runs/job-executions.js +4 -2
- package/dist/db/workflow-runs/job-executions.js.map +1 -1
- package/dist/db/workflow-runs/jobs.d.ts.map +1 -1
- package/dist/db/workflow-runs/jobs.js +29 -3
- package/dist/db/workflow-runs/jobs.js.map +1 -1
- package/dist/presentation/routes/project-access.d.ts +1 -1
- package/dist/temporal/activities/index.d.ts +1 -1
- package/dist/temporal/activities/index.d.ts.map +1 -1
- package/dist/temporal/activities/orchestration-activities.d.ts +1 -1
- package/dist/temporal/activities/orchestration-activities.d.ts.map +1 -1
- package/dist/temporal/workflows/test-env.d.ts +2 -2
- package/dist/tsconfig.test.tsbuildinfo +1 -1
- package/package.json +16 -16
- package/src/core/errors.ts +35 -0
- package/src/core/step-config/assemble-run-context.test.ts +232 -20
- package/src/core/step-config/assemble-run-context.ts +126 -28
- package/src/core/step-config/fields.ts +42 -3
- package/src/core/step-config/index.ts +2 -0
- package/src/core/step-config/job-output-limits.ts +130 -0
- package/src/core/step-config/materialize-workflow-model.test.ts +306 -1
- package/src/core/step-config/materialize-workflow-model.ts +60 -15
- package/src/db/job-listeners.ts +14 -3
- package/src/db/workflow-runs/job-executions.test.ts +207 -1
- package/src/db/workflow-runs/job-executions.ts +15 -2
- package/src/db/workflow-runs/jobs.test.ts +49 -0
- package/src/db/workflow-runs/jobs.ts +46 -6
- package/src/db/workflow-runs/run-create.test.ts +56 -0
- package/test/factories/workflow-model.ts +2 -0
- package/tsconfig.build.tsbuildinfo +1 -1
|
@@ -7,10 +7,24 @@ import {
|
|
|
7
7
|
import {parseWorkflowTemplate, planInterpolationField} from '@shipfox/expression';
|
|
8
8
|
import {createInterModuleKnownError} from '@shipfox/inter-module';
|
|
9
9
|
import type {AgentDefaultsResolver} from '#core/agent-defaults.js';
|
|
10
|
-
import {
|
|
10
|
+
import {
|
|
11
|
+
AgentConfigUnresolvableError,
|
|
12
|
+
InterpolationUnresolvableError,
|
|
13
|
+
JobOutputNotJsonSafeError,
|
|
14
|
+
JobOutputTooLargeError,
|
|
15
|
+
JobOutputTooManyEntriesError,
|
|
16
|
+
} from '#core/errors.js';
|
|
11
17
|
import {resolveTestAgentDefaults} from '#test/fixtures/agent-inter-module.js';
|
|
12
18
|
import {workflowModel} from '#test/index.js';
|
|
13
19
|
import {
|
|
20
|
+
MAX_JOB_OUTPUT_ENTRIES,
|
|
21
|
+
MAX_JOB_OUTPUT_NESTING_DEPTH,
|
|
22
|
+
MAX_JOB_OUTPUT_VALUE_BYTES,
|
|
23
|
+
MAX_JOB_OUTPUTS_TOTAL_BYTES,
|
|
24
|
+
normalizeJobOutputValue,
|
|
25
|
+
} from './job-output-limits.js';
|
|
26
|
+
import {
|
|
27
|
+
materializeJobOutputs,
|
|
14
28
|
materializeJobRunner,
|
|
15
29
|
materializeWorkflowModel as materializeWorkflowModelImpl,
|
|
16
30
|
modelHasAgentStep,
|
|
@@ -71,6 +85,38 @@ function creationContext(
|
|
|
71
85
|
return {site: 'run-creation', values};
|
|
72
86
|
}
|
|
73
87
|
|
|
88
|
+
function outputJob(
|
|
89
|
+
outputs: Readonly<Record<string, string>>,
|
|
90
|
+
outputTypes?: WorkflowModel['jobs'][number]['outputTypes'],
|
|
91
|
+
): WorkflowModel['jobs'][number] {
|
|
92
|
+
const model = workflowModel({
|
|
93
|
+
jobs: {
|
|
94
|
+
build: {
|
|
95
|
+
steps: [{key: 'collect', run: 'echo collect'}],
|
|
96
|
+
outputs,
|
|
97
|
+
...(outputTypes === undefined ? {} : {outputTypes}),
|
|
98
|
+
},
|
|
99
|
+
},
|
|
100
|
+
});
|
|
101
|
+
const job = model.jobs[0];
|
|
102
|
+
if (!job) throw new Error('Test model created no jobs');
|
|
103
|
+
return job;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function outputContext(outputs: Record<string, unknown>): WorkflowEvaluationContext {
|
|
107
|
+
return {
|
|
108
|
+
site: 'execution-resolution',
|
|
109
|
+
values: {
|
|
110
|
+
steps: {
|
|
111
|
+
collect: {
|
|
112
|
+
status: 'succeeded',
|
|
113
|
+
outputs,
|
|
114
|
+
},
|
|
115
|
+
},
|
|
116
|
+
},
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
|
|
74
120
|
describe('materializeWorkflowModel', () => {
|
|
75
121
|
it('keeps literal job names in runtime jobs', async () => {
|
|
76
122
|
const model = workflowModel({
|
|
@@ -1429,3 +1475,262 @@ describe('materializeJobRunner', () => {
|
|
|
1429
1475
|
expect(act).toThrow('Job runner labels are invalid: not valid');
|
|
1430
1476
|
});
|
|
1431
1477
|
});
|
|
1478
|
+
|
|
1479
|
+
describe('materializeJobOutputs', () => {
|
|
1480
|
+
it('preserves structured values from typed single-expression mappings', () => {
|
|
1481
|
+
const value = [{severity: 'high'}, {severity: 'medium'}];
|
|
1482
|
+
const job = outputJob(
|
|
1483
|
+
{findings: template('steps.collect.outputs.findings')},
|
|
1484
|
+
{
|
|
1485
|
+
findings: {
|
|
1486
|
+
kind: 'list',
|
|
1487
|
+
element: {kind: 'object', fields: {severity: 'string'}},
|
|
1488
|
+
},
|
|
1489
|
+
},
|
|
1490
|
+
);
|
|
1491
|
+
|
|
1492
|
+
const result = materializeJobOutputs({
|
|
1493
|
+
job,
|
|
1494
|
+
context: outputContext({findings: value}),
|
|
1495
|
+
definitionId: 'definition-1',
|
|
1496
|
+
});
|
|
1497
|
+
|
|
1498
|
+
expect(result).toEqual({findings: value});
|
|
1499
|
+
});
|
|
1500
|
+
|
|
1501
|
+
it('preserves number and boolean values from typed mappings', () => {
|
|
1502
|
+
const job = outputJob(
|
|
1503
|
+
{
|
|
1504
|
+
count: template('steps.collect.outputs.count'),
|
|
1505
|
+
ready: template('steps.collect.outputs.ready'),
|
|
1506
|
+
},
|
|
1507
|
+
{count: 'double', ready: 'bool'},
|
|
1508
|
+
);
|
|
1509
|
+
|
|
1510
|
+
const result = materializeJobOutputs({
|
|
1511
|
+
job,
|
|
1512
|
+
context: outputContext({count: 42, ready: true}),
|
|
1513
|
+
definitionId: 'definition-1',
|
|
1514
|
+
});
|
|
1515
|
+
|
|
1516
|
+
expect(result).toEqual({count: 42, ready: true});
|
|
1517
|
+
});
|
|
1518
|
+
|
|
1519
|
+
it('normalizes typed integer values into JSON-safe values', () => {
|
|
1520
|
+
const job = outputJob(
|
|
1521
|
+
{
|
|
1522
|
+
count: template('steps.collect.outputs.count'),
|
|
1523
|
+
preciseCount: template('steps.collect.outputs.preciseCount'),
|
|
1524
|
+
},
|
|
1525
|
+
{count: 'int', preciseCount: 'int'},
|
|
1526
|
+
);
|
|
1527
|
+
|
|
1528
|
+
const result = materializeJobOutputs({
|
|
1529
|
+
job,
|
|
1530
|
+
context: outputContext({count: 42n, preciseCount: 9007199254740993n}),
|
|
1531
|
+
definitionId: 'definition-1',
|
|
1532
|
+
});
|
|
1533
|
+
|
|
1534
|
+
expect(result).toEqual({count: 42, preciseCount: '9007199254740993'});
|
|
1535
|
+
expect(JSON.parse(JSON.stringify(result))).toEqual(result);
|
|
1536
|
+
});
|
|
1537
|
+
|
|
1538
|
+
it('normalizes typed timestamps into JSON-safe ISO strings', () => {
|
|
1539
|
+
const job = outputJob(
|
|
1540
|
+
{createdAt: template('steps.collect.outputs.createdAt')},
|
|
1541
|
+
{createdAt: 'timestamp'},
|
|
1542
|
+
);
|
|
1543
|
+
|
|
1544
|
+
const result = materializeJobOutputs({
|
|
1545
|
+
job,
|
|
1546
|
+
context: outputContext({createdAt: new Date('2026-08-06T12:34:56.000Z')}),
|
|
1547
|
+
definitionId: 'definition-1',
|
|
1548
|
+
});
|
|
1549
|
+
|
|
1550
|
+
expect(result).toEqual({createdAt: '2026-08-06T12:34:56.000Z'});
|
|
1551
|
+
expect(JSON.parse(JSON.stringify(result))).toEqual(result);
|
|
1552
|
+
});
|
|
1553
|
+
|
|
1554
|
+
it('stringifies mixed templates', () => {
|
|
1555
|
+
const job = outputJob({value: `prefix-${template('steps.collect.outputs.value')}`});
|
|
1556
|
+
|
|
1557
|
+
const result = materializeJobOutputs({
|
|
1558
|
+
job,
|
|
1559
|
+
context: outputContext({value: 42}),
|
|
1560
|
+
definitionId: 'definition-1',
|
|
1561
|
+
});
|
|
1562
|
+
|
|
1563
|
+
expect(result).toEqual({value: 'prefix-42'});
|
|
1564
|
+
});
|
|
1565
|
+
|
|
1566
|
+
it('stringifies outputs whose inferred type is string', () => {
|
|
1567
|
+
const job = outputJob({value: template('steps.collect.outputs.value')}, {value: 'string'});
|
|
1568
|
+
|
|
1569
|
+
const result = materializeJobOutputs({
|
|
1570
|
+
job,
|
|
1571
|
+
context: outputContext({value: 42}),
|
|
1572
|
+
definitionId: 'definition-1',
|
|
1573
|
+
});
|
|
1574
|
+
|
|
1575
|
+
expect(result).toEqual({value: '42'});
|
|
1576
|
+
});
|
|
1577
|
+
|
|
1578
|
+
it('stringifies exact mappings without an inferred type', () => {
|
|
1579
|
+
const job = outputJob({value: template('steps.collect.outputs.value')});
|
|
1580
|
+
|
|
1581
|
+
const result = materializeJobOutputs({
|
|
1582
|
+
job,
|
|
1583
|
+
context: outputContext({value: {severity: 'high'}}),
|
|
1584
|
+
definitionId: 'definition-1',
|
|
1585
|
+
});
|
|
1586
|
+
|
|
1587
|
+
expect(result).toEqual({value: '{"severity":"high"}'});
|
|
1588
|
+
});
|
|
1589
|
+
|
|
1590
|
+
it('keeps prototype-named untyped outputs as JSON strings', () => {
|
|
1591
|
+
const outputs = Object.fromEntries([
|
|
1592
|
+
['toString', template('steps.collect.outputs.value')],
|
|
1593
|
+
['__proto__', template('steps.collect.outputs.value')],
|
|
1594
|
+
]);
|
|
1595
|
+
|
|
1596
|
+
const result = materializeJobOutputs({
|
|
1597
|
+
job: outputJob(outputs),
|
|
1598
|
+
context: outputContext({value: {value: 'structured'}}),
|
|
1599
|
+
definitionId: 'definition-1',
|
|
1600
|
+
});
|
|
1601
|
+
|
|
1602
|
+
expect(Object.keys(result ?? {})).toEqual(['toString', '__proto__']);
|
|
1603
|
+
expect(Object.getOwnPropertyDescriptor(result ?? {}, 'toString')?.value).toBe(
|
|
1604
|
+
'{"value":"structured"}',
|
|
1605
|
+
);
|
|
1606
|
+
expect(Object.getOwnPropertyDescriptor(result ?? {}, '__proto__')?.value).toBe(
|
|
1607
|
+
'{"value":"structured"}',
|
|
1608
|
+
);
|
|
1609
|
+
});
|
|
1610
|
+
|
|
1611
|
+
it('measures structured values as JSON bytes for the per-value cap', () => {
|
|
1612
|
+
const job = outputJob(
|
|
1613
|
+
{payload: template('steps.collect.outputs.payload')},
|
|
1614
|
+
{payload: {kind: 'list', element: 'string'}},
|
|
1615
|
+
);
|
|
1616
|
+
const payload = ['x'.repeat(MAX_JOB_OUTPUT_VALUE_BYTES - 1)];
|
|
1617
|
+
|
|
1618
|
+
const materialize = () =>
|
|
1619
|
+
materializeJobOutputs({
|
|
1620
|
+
job,
|
|
1621
|
+
context: outputContext({payload}),
|
|
1622
|
+
definitionId: 'definition-1',
|
|
1623
|
+
});
|
|
1624
|
+
|
|
1625
|
+
expect(materialize).toThrow(JobOutputTooLargeError);
|
|
1626
|
+
expect(materialize).toThrow(`Job output "payload" exceeds the ${MAX_JOB_OUTPUT_VALUE_BYTES}`);
|
|
1627
|
+
});
|
|
1628
|
+
|
|
1629
|
+
it('measures string values as UTF-8 bytes for the per-value cap', () => {
|
|
1630
|
+
const job = outputJob({payload: template('steps.collect.outputs.payload')});
|
|
1631
|
+
const payload = 'é'.repeat(Math.ceil(MAX_JOB_OUTPUT_VALUE_BYTES / 2) + 1);
|
|
1632
|
+
|
|
1633
|
+
const materialize = () =>
|
|
1634
|
+
materializeJobOutputs({
|
|
1635
|
+
job,
|
|
1636
|
+
context: outputContext({payload}),
|
|
1637
|
+
definitionId: 'definition-1',
|
|
1638
|
+
});
|
|
1639
|
+
|
|
1640
|
+
expect(materialize).toThrow(JobOutputTooLargeError);
|
|
1641
|
+
});
|
|
1642
|
+
|
|
1643
|
+
it('rejects the total materialized job output cap', () => {
|
|
1644
|
+
const keys = ['one', 'two', 'three', 'four', 'five'];
|
|
1645
|
+
const outputs = Object.fromEntries(
|
|
1646
|
+
keys.map((key) => [key, template(`steps.collect.outputs.${key}`)]),
|
|
1647
|
+
);
|
|
1648
|
+
const values = Object.fromEntries(
|
|
1649
|
+
keys.map((key) => [key, 'x'.repeat(MAX_JOB_OUTPUT_VALUE_BYTES)]),
|
|
1650
|
+
);
|
|
1651
|
+
const job = outputJob(outputs);
|
|
1652
|
+
|
|
1653
|
+
const materialize = () =>
|
|
1654
|
+
materializeJobOutputs({job, context: outputContext(values), definitionId: 'definition-1'});
|
|
1655
|
+
|
|
1656
|
+
expect(materialize).toThrow(JobOutputTooLargeError);
|
|
1657
|
+
expect(materialize).toThrow(
|
|
1658
|
+
`Job outputs exceed the ${MAX_JOB_OUTPUTS_TOTAL_BYTES}-byte total limit (at "four")`,
|
|
1659
|
+
);
|
|
1660
|
+
});
|
|
1661
|
+
|
|
1662
|
+
it('counts output keys and JSON encoding overhead against the total cap', () => {
|
|
1663
|
+
const keys = ['a', 'b', 'c', 'd'].map((suffix) => `output${suffix}${'x'.repeat(256)}`);
|
|
1664
|
+
const outputs = Object.fromEntries(
|
|
1665
|
+
keys.map((key) => [key, template(`steps.collect.outputs.${key}`)]),
|
|
1666
|
+
);
|
|
1667
|
+
const values = Object.fromEntries(
|
|
1668
|
+
keys.map((key) => [key, 'x'.repeat(MAX_JOB_OUTPUT_VALUE_BYTES - 16)]),
|
|
1669
|
+
);
|
|
1670
|
+
const job = outputJob(outputs);
|
|
1671
|
+
|
|
1672
|
+
const materialize = () =>
|
|
1673
|
+
materializeJobOutputs({job, context: outputContext(values), definitionId: 'definition-1'});
|
|
1674
|
+
|
|
1675
|
+
expect(materialize).toThrow(JobOutputTooLargeError);
|
|
1676
|
+
expect(materialize).toThrow(
|
|
1677
|
+
`Job outputs exceed the ${MAX_JOB_OUTPUTS_TOTAL_BYTES}-byte total limit`,
|
|
1678
|
+
);
|
|
1679
|
+
});
|
|
1680
|
+
|
|
1681
|
+
it('rejects job output maps with too many entries', () => {
|
|
1682
|
+
const outputs = Object.fromEntries(
|
|
1683
|
+
Array.from({length: MAX_JOB_OUTPUT_ENTRIES + 1}, (_, index) => {
|
|
1684
|
+
const key = `output${index}`;
|
|
1685
|
+
return [key, template(`steps.collect.outputs.${key}`)];
|
|
1686
|
+
}),
|
|
1687
|
+
);
|
|
1688
|
+
const job = outputJob(outputs);
|
|
1689
|
+
|
|
1690
|
+
const materialize = () =>
|
|
1691
|
+
materializeJobOutputs({job, context: outputContext({}), definitionId: 'definition-1'});
|
|
1692
|
+
|
|
1693
|
+
expect(materialize).toThrow(JobOutputTooManyEntriesError);
|
|
1694
|
+
expect(materialize).toThrow(
|
|
1695
|
+
`Job outputs cannot define more than ${MAX_JOB_OUTPUT_ENTRIES} entries`,
|
|
1696
|
+
);
|
|
1697
|
+
});
|
|
1698
|
+
|
|
1699
|
+
it('rejects unsupported values with a JSON persistence error', () => {
|
|
1700
|
+
const circular: Record<string, unknown> = {};
|
|
1701
|
+
circular.self = circular;
|
|
1702
|
+
|
|
1703
|
+
expect(() => normalizeJobOutputValue(circular, 'payload')).toThrow(JobOutputNotJsonSafeError);
|
|
1704
|
+
expect(() => normalizeJobOutputValue(circular, 'payload')).toThrow(
|
|
1705
|
+
'Job output "payload" cannot be persisted as JSON',
|
|
1706
|
+
);
|
|
1707
|
+
});
|
|
1708
|
+
|
|
1709
|
+
it('rejects deeply nested values with a JSON persistence error', () => {
|
|
1710
|
+
let value: unknown = 'leaf';
|
|
1711
|
+
for (let index = 0; index <= MAX_JOB_OUTPUT_NESTING_DEPTH; index += 1) {
|
|
1712
|
+
value = {value};
|
|
1713
|
+
}
|
|
1714
|
+
|
|
1715
|
+
expect(() => normalizeJobOutputValue(value, 'payload')).toThrow(JobOutputNotJsonSafeError);
|
|
1716
|
+
expect(() => normalizeJobOutputValue(value, 'payload')).toThrow(
|
|
1717
|
+
`values cannot be nested deeper than ${MAX_JOB_OUTPUT_NESTING_DEPTH} levels`,
|
|
1718
|
+
);
|
|
1719
|
+
});
|
|
1720
|
+
|
|
1721
|
+
it('fails when a typed output path is missing', () => {
|
|
1722
|
+
const job = outputJob(
|
|
1723
|
+
{findings: template('steps.collect.outputs.findings')},
|
|
1724
|
+
{findings: {kind: 'list', element: 'string'}},
|
|
1725
|
+
);
|
|
1726
|
+
|
|
1727
|
+
const materialize = () =>
|
|
1728
|
+
materializeJobOutputs({
|
|
1729
|
+
job,
|
|
1730
|
+
context: outputContext({}),
|
|
1731
|
+
definitionId: 'definition-1',
|
|
1732
|
+
});
|
|
1733
|
+
|
|
1734
|
+
expect(materialize).toThrow(InterpolationUnresolvableError);
|
|
1735
|
+
});
|
|
1736
|
+
});
|
|
@@ -5,8 +5,21 @@ import type {
|
|
|
5
5
|
AgentToolMaterializationContext,
|
|
6
6
|
AgentToolMaterializationSnapshot,
|
|
7
7
|
} from '#core/agent-tools.js';
|
|
8
|
-
import {
|
|
9
|
-
|
|
8
|
+
import {
|
|
9
|
+
InvalidJobRunnerLabelsError,
|
|
10
|
+
JobOutputTooLargeError,
|
|
11
|
+
JobOutputTooManyEntriesError,
|
|
12
|
+
} from '#core/errors.js';
|
|
13
|
+
import {completeStepField, completeStepFieldWithType} from './fields.js';
|
|
14
|
+
import {
|
|
15
|
+
type JsonSafeJobOutputValue,
|
|
16
|
+
jobOutputRecordEntryByteLength,
|
|
17
|
+
jobOutputValueByteLength,
|
|
18
|
+
MAX_JOB_OUTPUT_ENTRIES,
|
|
19
|
+
MAX_JOB_OUTPUT_VALUE_BYTES,
|
|
20
|
+
MAX_JOB_OUTPUTS_TOTAL_BYTES,
|
|
21
|
+
normalizeJobOutputValue,
|
|
22
|
+
} from './job-output-limits.js';
|
|
10
23
|
import {
|
|
11
24
|
type MaterializedWorkflowStep,
|
|
12
25
|
materializeJobExecutionSteps,
|
|
@@ -111,22 +124,54 @@ export function materializeJobOutputs(params: {
|
|
|
111
124
|
readonly job: WorkflowModelJob;
|
|
112
125
|
readonly context: WorkflowEvaluationContext;
|
|
113
126
|
readonly definitionId: string;
|
|
114
|
-
}): Record<string,
|
|
127
|
+
}): Record<string, unknown> | null {
|
|
115
128
|
const outputs = params.job.outputs;
|
|
116
129
|
if (outputs === undefined) return null;
|
|
117
130
|
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
131
|
+
const outputEntries = Object.entries(outputs);
|
|
132
|
+
if (outputEntries.length > MAX_JOB_OUTPUT_ENTRIES) {
|
|
133
|
+
throw new JobOutputTooManyEntriesError(outputEntries.length, MAX_JOB_OUTPUT_ENTRIES);
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
const materialized: Record<string, JsonSafeJobOutputValue> = {};
|
|
137
|
+
// This is exactly JSON.stringify(materialized): start with "{}", then add each
|
|
138
|
+
// key/value pair and its comma (except for the first entry).
|
|
139
|
+
let recordBytes = 2;
|
|
140
|
+
|
|
141
|
+
for (const [index, [key, template]] of outputEntries.entries()) {
|
|
142
|
+
const outputTypes = params.job.outputTypes;
|
|
143
|
+
const outputType =
|
|
144
|
+
outputTypes !== undefined && Object.hasOwn(outputTypes, key) ? outputTypes[key] : undefined;
|
|
145
|
+
const completionParams = {
|
|
146
|
+
field: 'job.outputs' as const,
|
|
147
|
+
errorField: 'job.outputs' as const,
|
|
148
|
+
template: {segments: template},
|
|
149
|
+
context: params.context,
|
|
150
|
+
definitionId: params.definitionId,
|
|
151
|
+
};
|
|
152
|
+
const value =
|
|
153
|
+
outputType === undefined || outputType === 'string'
|
|
154
|
+
? completeStepField(completionParams)
|
|
155
|
+
: completeStepFieldWithType(completionParams);
|
|
156
|
+
const normalizedValue = normalizeJobOutputValue(value, key);
|
|
157
|
+
const valueBytes = jobOutputValueByteLength(normalizedValue);
|
|
158
|
+
if (valueBytes > MAX_JOB_OUTPUT_VALUE_BYTES) {
|
|
159
|
+
throw new JobOutputTooLargeError(key, MAX_JOB_OUTPUT_VALUE_BYTES, 'value');
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
recordBytes += (index === 0 ? 0 : 1) + jobOutputRecordEntryByteLength(key, normalizedValue);
|
|
163
|
+
Object.defineProperty(materialized, key, {
|
|
164
|
+
configurable: true,
|
|
165
|
+
enumerable: true,
|
|
166
|
+
value: normalizedValue,
|
|
167
|
+
writable: true,
|
|
168
|
+
});
|
|
169
|
+
if (recordBytes > MAX_JOB_OUTPUTS_TOTAL_BYTES) {
|
|
170
|
+
throw new JobOutputTooLargeError(key, MAX_JOB_OUTPUTS_TOTAL_BYTES, 'total');
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
return materialized;
|
|
130
175
|
}
|
|
131
176
|
|
|
132
177
|
export function modelHasAgentStep(model: WorkflowModel): boolean {
|
package/src/db/job-listeners.ts
CHANGED
|
@@ -27,6 +27,7 @@ import {
|
|
|
27
27
|
applyListenerFilterSnapshots,
|
|
28
28
|
assembleListenerSnapshotContext,
|
|
29
29
|
type ListenerTriggerWithSnapshot,
|
|
30
|
+
listenerFilterOutputTypesForJobs,
|
|
30
31
|
planListenerFilterSnapshots,
|
|
31
32
|
} from '#core/step-config/assemble-run-context.js';
|
|
32
33
|
import {
|
|
@@ -73,8 +74,9 @@ type JobActivatedListenerMatcher = Extract<
|
|
|
73
74
|
function applyActivatedListenerFilterSnapshots(
|
|
74
75
|
plans: Parameters<typeof applyListenerFilterSnapshots>[0],
|
|
75
76
|
context: Parameters<typeof applyListenerFilterSnapshots>[1],
|
|
77
|
+
outputTypes: Parameters<typeof applyListenerFilterSnapshots>[2],
|
|
76
78
|
): JobActivatedListenerMatcher[] {
|
|
77
|
-
return applyListenerFilterSnapshots(plans, context);
|
|
79
|
+
return applyListenerFilterSnapshots(plans, context, outputTypes);
|
|
78
80
|
}
|
|
79
81
|
|
|
80
82
|
export async function activateJobListener(
|
|
@@ -150,6 +152,7 @@ export async function activateJobListener(
|
|
|
150
152
|
plan: snapshotPlan,
|
|
151
153
|
dependencyJobs,
|
|
152
154
|
});
|
|
155
|
+
const listenerOutputTypes = listenerFilterOutputTypesForJobs(dependencyJobs);
|
|
153
156
|
|
|
154
157
|
await writeWorkflowsOutboxEvent(tx, {
|
|
155
158
|
type: WORKFLOWS_JOB_ACTIVATED,
|
|
@@ -158,11 +161,19 @@ export async function activateJobListener(
|
|
|
158
161
|
workflowRunId: target.run.id,
|
|
159
162
|
workspaceId: target.run.workspaceId,
|
|
160
163
|
mode: 'listening',
|
|
161
|
-
on: applyActivatedListenerFilterSnapshots(
|
|
164
|
+
on: applyActivatedListenerFilterSnapshots(
|
|
165
|
+
snapshotPlan.on,
|
|
166
|
+
snapshotContext,
|
|
167
|
+
listenerOutputTypes,
|
|
168
|
+
),
|
|
162
169
|
until:
|
|
163
170
|
matchers.until === null
|
|
164
171
|
? null
|
|
165
|
-
: applyActivatedListenerFilterSnapshots(
|
|
172
|
+
: applyActivatedListenerFilterSnapshots(
|
|
173
|
+
snapshotPlan.until,
|
|
174
|
+
snapshotContext,
|
|
175
|
+
listenerOutputTypes,
|
|
176
|
+
),
|
|
166
177
|
},
|
|
167
178
|
});
|
|
168
179
|
}
|
|
@@ -1,14 +1,21 @@
|
|
|
1
1
|
import {WORKFLOWS_JOB_EXECUTION_TIMED_OUT} from '@shipfox/api-workflows-dto';
|
|
2
2
|
import {and, eq, sql} from 'drizzle-orm';
|
|
3
|
-
import {
|
|
3
|
+
import {
|
|
4
|
+
MAX_JOB_OUTPUT_ENTRIES,
|
|
5
|
+
MAX_JOB_OUTPUT_VALUE_BYTES,
|
|
6
|
+
} from '#core/step-config/job-output-limits.js';
|
|
7
|
+
import {buildModel, template, workflowRunAttemptId} from '#test/helpers/workflow-runs.js';
|
|
4
8
|
import {db} from '../db.js';
|
|
5
9
|
import {workflowsOutbox} from '../schema/outbox.js';
|
|
6
10
|
import {
|
|
11
|
+
applyStepResult,
|
|
7
12
|
createWorkflowRun,
|
|
8
13
|
failJobExecutionAsTimedOut,
|
|
14
|
+
finishStepAttempt,
|
|
9
15
|
getFirstJobExecutionByJobId,
|
|
10
16
|
getJobsByWorkflowRunId,
|
|
11
17
|
getStepsByJobId,
|
|
18
|
+
markStepRunning,
|
|
12
19
|
resolveJobExecutionAfterLeaseExpiry,
|
|
13
20
|
updateJobExecutionStatus,
|
|
14
21
|
} from '../workflow-runs.js';
|
|
@@ -96,4 +103,203 @@ describe('workflow run job executions', () => {
|
|
|
96
103
|
const jobSteps = await getStepsByJobId(job.id);
|
|
97
104
|
expect(jobSteps.every((step) => step.status === 'pending')).toBe(true);
|
|
98
105
|
});
|
|
106
|
+
|
|
107
|
+
test('persists a structured job output when execution succeeds', async () => {
|
|
108
|
+
const run = await createWorkflowRun({
|
|
109
|
+
workspaceId,
|
|
110
|
+
projectId,
|
|
111
|
+
definitionId,
|
|
112
|
+
model: buildModel({
|
|
113
|
+
jobs: {
|
|
114
|
+
build: {
|
|
115
|
+
steps: [{key: 'collect', run: 'echo build'}],
|
|
116
|
+
outputs: {findings: template('steps.collect.outputs.findings')},
|
|
117
|
+
outputTypes: {
|
|
118
|
+
findings: {
|
|
119
|
+
kind: 'list',
|
|
120
|
+
element: {kind: 'object', fields: {severity: 'string'}},
|
|
121
|
+
},
|
|
122
|
+
},
|
|
123
|
+
},
|
|
124
|
+
},
|
|
125
|
+
}),
|
|
126
|
+
triggerPayload: {
|
|
127
|
+
source: 'manual',
|
|
128
|
+
event: 'fire',
|
|
129
|
+
subscriptionId: crypto.randomUUID(),
|
|
130
|
+
userId: crypto.randomUUID(),
|
|
131
|
+
},
|
|
132
|
+
});
|
|
133
|
+
const [job] = await getJobsByWorkflowRunId(run.id);
|
|
134
|
+
if (!job) throw new Error('Expected workflow job');
|
|
135
|
+
const execution = await getFirstJobExecutionByJobId(job.id);
|
|
136
|
+
if (!execution) throw new Error('Expected job execution');
|
|
137
|
+
await finishCollectedStep(job.id, {findings: [{severity: 'high'}]});
|
|
138
|
+
|
|
139
|
+
const resolved = await updateJobExecutionStatus({
|
|
140
|
+
jobExecutionId: execution.id,
|
|
141
|
+
status: 'succeeded',
|
|
142
|
+
expectedVersion: execution.version,
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
expect(resolved.outputs).toEqual({findings: [{severity: 'high'}]});
|
|
146
|
+
expect(Array.isArray(resolved.outputs?.findings)).toBe(true);
|
|
147
|
+
const persisted = await getFirstJobExecutionByJobId(job.id);
|
|
148
|
+
expect(persisted?.outputs).toEqual({findings: [{severity: 'high'}]});
|
|
149
|
+
});
|
|
150
|
+
|
|
151
|
+
test('persists JSON-safe typed integer and timestamp outputs when execution succeeds', async () => {
|
|
152
|
+
const run = await createWorkflowRun({
|
|
153
|
+
workspaceId,
|
|
154
|
+
projectId,
|
|
155
|
+
definitionId,
|
|
156
|
+
model: buildModel({
|
|
157
|
+
jobs: {
|
|
158
|
+
build: {
|
|
159
|
+
steps: [{key: 'collect', run: 'echo build'}],
|
|
160
|
+
outputs: {
|
|
161
|
+
count: template('steps.collect.outputs.count'),
|
|
162
|
+
createdAt: template('run.created_at'),
|
|
163
|
+
},
|
|
164
|
+
outputTypes: {count: 'int', createdAt: 'timestamp'},
|
|
165
|
+
},
|
|
166
|
+
},
|
|
167
|
+
}),
|
|
168
|
+
triggerPayload: {
|
|
169
|
+
source: 'manual',
|
|
170
|
+
event: 'fire',
|
|
171
|
+
subscriptionId: crypto.randomUUID(),
|
|
172
|
+
userId: crypto.randomUUID(),
|
|
173
|
+
},
|
|
174
|
+
});
|
|
175
|
+
const [job] = await getJobsByWorkflowRunId(run.id);
|
|
176
|
+
if (!job) throw new Error('Expected workflow job');
|
|
177
|
+
const execution = await getFirstJobExecutionByJobId(job.id);
|
|
178
|
+
if (!execution) throw new Error('Expected job execution');
|
|
179
|
+
await finishCollectedStep(job.id, {count: 42});
|
|
180
|
+
|
|
181
|
+
const resolved = await updateJobExecutionStatus({
|
|
182
|
+
jobExecutionId: execution.id,
|
|
183
|
+
status: 'succeeded',
|
|
184
|
+
expectedVersion: execution.version,
|
|
185
|
+
});
|
|
186
|
+
|
|
187
|
+
expect(resolved.outputs).toEqual({count: 42, createdAt: run.createdAt.toISOString()});
|
|
188
|
+
const persisted = await getFirstJobExecutionByJobId(job.id);
|
|
189
|
+
expect(persisted?.outputs).toEqual({count: 42, createdAt: run.createdAt.toISOString()});
|
|
190
|
+
});
|
|
191
|
+
|
|
192
|
+
test('fails a successful execution when a materialized job output is too large', async () => {
|
|
193
|
+
const run = await createWorkflowRun({
|
|
194
|
+
workspaceId,
|
|
195
|
+
projectId,
|
|
196
|
+
definitionId,
|
|
197
|
+
model: buildModel({
|
|
198
|
+
jobs: {
|
|
199
|
+
build: {
|
|
200
|
+
steps: [{key: 'collect', run: 'echo build'}],
|
|
201
|
+
outputs: {payload: template('steps.collect.outputs.payload')},
|
|
202
|
+
outputTypes: {payload: {kind: 'list', element: 'string'}},
|
|
203
|
+
},
|
|
204
|
+
},
|
|
205
|
+
}),
|
|
206
|
+
triggerPayload: {
|
|
207
|
+
source: 'manual',
|
|
208
|
+
event: 'fire',
|
|
209
|
+
subscriptionId: crypto.randomUUID(),
|
|
210
|
+
userId: crypto.randomUUID(),
|
|
211
|
+
},
|
|
212
|
+
});
|
|
213
|
+
const [job] = await getJobsByWorkflowRunId(run.id);
|
|
214
|
+
if (!job) throw new Error('Expected workflow job');
|
|
215
|
+
const execution = await getFirstJobExecutionByJobId(job.id);
|
|
216
|
+
if (!execution) throw new Error('Expected job execution');
|
|
217
|
+
await finishCollectedStep(job.id, {
|
|
218
|
+
payload: ['x'.repeat(MAX_JOB_OUTPUT_VALUE_BYTES - 1)],
|
|
219
|
+
});
|
|
220
|
+
|
|
221
|
+
const resolved = await updateJobExecutionStatus({
|
|
222
|
+
jobExecutionId: execution.id,
|
|
223
|
+
status: 'succeeded',
|
|
224
|
+
expectedVersion: execution.version,
|
|
225
|
+
});
|
|
226
|
+
|
|
227
|
+
expect(resolved).toMatchObject({status: 'failed', statusReason: 'unknown', outputs: null});
|
|
228
|
+
});
|
|
229
|
+
|
|
230
|
+
test('fails a successful execution when the persisted model has too many job outputs', async () => {
|
|
231
|
+
const outputs = Object.fromEntries(
|
|
232
|
+
Array.from({length: MAX_JOB_OUTPUT_ENTRIES + 1}, (_, index) => {
|
|
233
|
+
const key = `output${index}`;
|
|
234
|
+
return [key, template(`steps.collect.outputs.${key}`)];
|
|
235
|
+
}),
|
|
236
|
+
);
|
|
237
|
+
const run = await createWorkflowRun({
|
|
238
|
+
workspaceId,
|
|
239
|
+
projectId,
|
|
240
|
+
definitionId,
|
|
241
|
+
model: buildModel({
|
|
242
|
+
jobs: {
|
|
243
|
+
build: {
|
|
244
|
+
steps: [{key: 'collect', run: 'echo build'}],
|
|
245
|
+
outputs,
|
|
246
|
+
},
|
|
247
|
+
},
|
|
248
|
+
}),
|
|
249
|
+
triggerPayload: {
|
|
250
|
+
source: 'manual',
|
|
251
|
+
event: 'fire',
|
|
252
|
+
subscriptionId: crypto.randomUUID(),
|
|
253
|
+
userId: crypto.randomUUID(),
|
|
254
|
+
},
|
|
255
|
+
});
|
|
256
|
+
const [job] = await getJobsByWorkflowRunId(run.id);
|
|
257
|
+
if (!job) throw new Error('Expected workflow job');
|
|
258
|
+
const execution = await getFirstJobExecutionByJobId(job.id);
|
|
259
|
+
if (!execution) throw new Error('Expected job execution');
|
|
260
|
+
await finishCollectedStep(job.id, {});
|
|
261
|
+
|
|
262
|
+
const resolved = await updateJobExecutionStatus({
|
|
263
|
+
jobExecutionId: execution.id,
|
|
264
|
+
status: 'succeeded',
|
|
265
|
+
expectedVersion: execution.version,
|
|
266
|
+
});
|
|
267
|
+
|
|
268
|
+
expect(resolved).toMatchObject({status: 'failed', statusReason: 'unknown', outputs: null});
|
|
269
|
+
});
|
|
99
270
|
});
|
|
271
|
+
|
|
272
|
+
async function finishCollectedStep(jobId: string, output: Record<string, unknown>): Promise<void> {
|
|
273
|
+
const steps = await getStepsByJobId(jobId);
|
|
274
|
+
const step = steps.find((candidate) => candidate.key === 'collect');
|
|
275
|
+
if (!step) throw new Error('Expected collect step');
|
|
276
|
+
|
|
277
|
+
await db().transaction(async (tx) => {
|
|
278
|
+
const running = await markStepRunning(
|
|
279
|
+
{jobExecutionId: step.jobExecutionId, stepId: step.id},
|
|
280
|
+
tx,
|
|
281
|
+
);
|
|
282
|
+
if (!running) throw new Error('Expected pending collect step');
|
|
283
|
+
await finishStepAttempt(
|
|
284
|
+
{
|
|
285
|
+
stepId: step.id,
|
|
286
|
+
attempt: running.currentAttempt,
|
|
287
|
+
status: 'succeeded',
|
|
288
|
+
output,
|
|
289
|
+
response: null,
|
|
290
|
+
exitCode: 0,
|
|
291
|
+
logOutcome: 'drained',
|
|
292
|
+
},
|
|
293
|
+
tx,
|
|
294
|
+
);
|
|
295
|
+
await applyStepResult(
|
|
296
|
+
{
|
|
297
|
+
jobExecutionId: step.jobExecutionId,
|
|
298
|
+
stepId: step.id,
|
|
299
|
+
status: 'succeeded',
|
|
300
|
+
error: null,
|
|
301
|
+
},
|
|
302
|
+
tx,
|
|
303
|
+
);
|
|
304
|
+
});
|
|
305
|
+
}
|