@octaviaflow/flow-rules 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/catalog/general/aggregate/config.d.ts +126 -0
- package/dist/catalog/general/aggregate/index.d.ts +11 -0
- package/dist/catalog/index.d.ts +2 -1
- package/dist/catalog/isOneOf.d.ts +7 -0
- package/dist/catalog/logic/for-each/recordSchema.d.ts +26 -0
- package/dist/catalog/types.d.ts +1 -0
- package/dist/index.cjs +360 -2
- package/dist/index.d.ts +2 -0
- package/dist/index.js +360 -2
- package/dist/rules/configContracts.d.ts +2 -2
- package/package.json +1 -1
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Aggregate Action — Config Contract
|
|
3
|
+
*
|
|
4
|
+
* Fold the upstream records into ONE row — or one row per group — of computed
|
|
5
|
+
* values. It exists because nothing else could total a column (DEV-422): a
|
|
6
|
+
* Transform maps one record at a time and cannot see the others, and the only
|
|
7
|
+
* built-in sum was ForEach's `collect: 'reduce'`, which reduces loop-body
|
|
8
|
+
* results and is reachable from nowhere an author, or an agent reading
|
|
9
|
+
* `describe_action_type`, would look.
|
|
10
|
+
*
|
|
11
|
+
* Output: `{ data: [ { ...groupKeys, ...metrics } ], _meta }` — the same
|
|
12
|
+
* `data: row[]` envelope Query, Transform and Variables emit, so the Transform
|
|
13
|
+
* after it maps `{{totals[].TotalAmount}}` exactly as it maps a Query's rows.
|
|
14
|
+
*
|
|
15
|
+
* The Engine's `aggregateExecutor.ts` reads this shape; the two stay in step.
|
|
16
|
+
*/
|
|
17
|
+
/** The functions a metric can apply. `describe_action_type` lists them. */
|
|
18
|
+
export declare const AGGREGATE_FUNCTIONS: readonly ["sum", "count", "avg", "min", "max", "count_distinct", "first", "last"];
|
|
19
|
+
export type AggregateFn = (typeof AGGREGATE_FUNCTIONS)[number];
|
|
20
|
+
/** What each function computes — the panel's help text and the contract's description. */
|
|
21
|
+
export declare const AGGREGATE_FUNCTION_DESCRIPTIONS: Readonly<Record<AggregateFn, string>>;
|
|
22
|
+
export interface AggregateMetric {
|
|
23
|
+
/** One of `AGGREGATE_FUNCTIONS`. */
|
|
24
|
+
fn: AggregateFn;
|
|
25
|
+
/**
|
|
26
|
+
* The record field it reads — a dot-path such as "Balance" or
|
|
27
|
+
* "CustomerRef.name". `count` may omit it to count records.
|
|
28
|
+
*/
|
|
29
|
+
field?: string;
|
|
30
|
+
/** The key the value lands under on the output row. An identifier, unique within the step. */
|
|
31
|
+
as: string;
|
|
32
|
+
/** Round a sum or avg to this many decimal places (0–10). Omit to keep the value as computed. */
|
|
33
|
+
decimals?: number;
|
|
34
|
+
}
|
|
35
|
+
export interface AggregateConfigType {
|
|
36
|
+
/**
|
|
37
|
+
* Which upstream records: a path to an array in the upstream step's output.
|
|
38
|
+
* Empty means the records the previous step produced (its `data`). Resolved
|
|
39
|
+
* exactly as ForEach's `arrayPath`.
|
|
40
|
+
*/
|
|
41
|
+
arrayPath?: string;
|
|
42
|
+
/**
|
|
43
|
+
* Record fields (dot-paths) to group by. Empty: one row for all records.
|
|
44
|
+
* Each key lands on the output row under its last segment
|
|
45
|
+
* ("CustomerRef.name" → "name").
|
|
46
|
+
*/
|
|
47
|
+
groupBy?: string[];
|
|
48
|
+
/** The values to compute. At least one. */
|
|
49
|
+
metrics: AggregateMetric[];
|
|
50
|
+
/**
|
|
51
|
+
* The name later steps use for this result, e.g. `totals` →
|
|
52
|
+
* `{{totals[].TotalAmount}}`. Defaults to the step's name.
|
|
53
|
+
*/
|
|
54
|
+
outputAs?: string;
|
|
55
|
+
/**
|
|
56
|
+
* The output ROW's schema, written by the panel at save time so the picker
|
|
57
|
+
* shows the keys before a run. The Backend contributor derives the same
|
|
58
|
+
* thing from the config with `computeOutputSchema`.
|
|
59
|
+
*/
|
|
60
|
+
_outputSchema?: {
|
|
61
|
+
type: 'object';
|
|
62
|
+
properties: Record<string, unknown>;
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
/** A JSON-schema-like node, as the DataStream and `_outputSchema` carry it. */
|
|
66
|
+
export interface AggregateSchemaNode {
|
|
67
|
+
type?: string;
|
|
68
|
+
properties?: Record<string, AggregateSchemaNode>;
|
|
69
|
+
items?: AggregateSchemaNode;
|
|
70
|
+
description?: string;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* The output key a group-by field lands under: its last path segment, with a
|
|
74
|
+
* trailing `[]` removed. "CustomerRef.name" → "name"; "Region" → "Region".
|
|
75
|
+
* The Engine names the group keys the same way.
|
|
76
|
+
*/
|
|
77
|
+
export declare function groupKeyName(path: string): string;
|
|
78
|
+
export declare const AggregateConfig: {
|
|
79
|
+
schema: {
|
|
80
|
+
arrayPath: {
|
|
81
|
+
type: string;
|
|
82
|
+
required: boolean;
|
|
83
|
+
dynamic: boolean;
|
|
84
|
+
description: string;
|
|
85
|
+
};
|
|
86
|
+
groupBy: {
|
|
87
|
+
type: string;
|
|
88
|
+
required: boolean;
|
|
89
|
+
description: string;
|
|
90
|
+
};
|
|
91
|
+
metrics: {
|
|
92
|
+
type: string;
|
|
93
|
+
required: boolean;
|
|
94
|
+
description: string;
|
|
95
|
+
};
|
|
96
|
+
outputAs: {
|
|
97
|
+
type: string;
|
|
98
|
+
required: boolean;
|
|
99
|
+
description: string;
|
|
100
|
+
};
|
|
101
|
+
};
|
|
102
|
+
defaults: AggregateConfigType;
|
|
103
|
+
/**
|
|
104
|
+
* What blocks execution, refused before the run (a 409 on an active flow
|
|
105
|
+
* over MCP, a warning on a draft): no metrics, an unknown function, a
|
|
106
|
+
* function with nothing to read, an output key that is not an identifier or
|
|
107
|
+
* is taken, decimals outside 0–10, an outputAs that is not a single name.
|
|
108
|
+
*/
|
|
109
|
+
validate(config: Partial<AggregateConfigType>): {
|
|
110
|
+
valid: boolean;
|
|
111
|
+
errors: string[];
|
|
112
|
+
};
|
|
113
|
+
/**
|
|
114
|
+
* The schema of ONE output row: the group keys (typed from the source record
|
|
115
|
+
* when `source` is known) and each metric under its `as`. The panel writes
|
|
116
|
+
* it to `_outputSchema`; the Backend contributor wraps it as `array<row>` at
|
|
117
|
+
* the step's output path. One implementation, two callers.
|
|
118
|
+
*/
|
|
119
|
+
computeOutputSchema(config: Partial<AggregateConfigType>, source?: AggregateSchemaNode): {
|
|
120
|
+
type: "object";
|
|
121
|
+
properties: Record<string, unknown>;
|
|
122
|
+
};
|
|
123
|
+
merge(userConfig: Partial<AggregateConfigType>): AggregateConfigType;
|
|
124
|
+
/** "sum(Balance), count by name" — the node's auto label. */
|
|
125
|
+
getLabel(config: Partial<AggregateConfigType>): string;
|
|
126
|
+
};
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Aggregate Action
|
|
3
|
+
*
|
|
4
|
+
* Totals, counts, averages, min/max, distinct counts and first/last over the
|
|
5
|
+
* upstream records — one output row, or one per group. Sits after the step
|
|
6
|
+
* that produces the rows and before the step that writes the result (the
|
|
7
|
+
* "TOTAL row" case: Query → Transform (insert rows) → Aggregate → Transform
|
|
8
|
+
* (insert one row)). The config contract is `./config.ts` (DEV-422).
|
|
9
|
+
*/
|
|
10
|
+
import { ActionDefinition } from '../../types';
|
|
11
|
+
export declare const aggregateDefinition: ActionDefinition;
|
package/dist/catalog/index.d.ts
CHANGED
|
@@ -16,6 +16,7 @@ import { forEachDefinition } from './logic/for-each';
|
|
|
16
16
|
import { groupDefinition } from './logic/group';
|
|
17
17
|
import { queryDefinition } from './general/query';
|
|
18
18
|
import { transformDefinition } from './general/transform';
|
|
19
|
+
import { aggregateDefinition } from './general/aggregate';
|
|
19
20
|
import { csvActionDefinition } from './storage/csv-action';
|
|
20
21
|
import { jsonActionDefinition } from './storage/json-action';
|
|
21
22
|
import { xmlActionDefinition } from './storage/xml-action';
|
|
@@ -54,4 +55,4 @@ export declare const getActionById: (id: string) => ActionDefinition | undefined
|
|
|
54
55
|
* Get actions by category
|
|
55
56
|
*/
|
|
56
57
|
export declare const getActionsByCategory: (category: ActionCategory) => ActionDefinition[];
|
|
57
|
-
export { csvActionDefinition, jsonActionDefinition, xmlActionDefinition, excelActionDefinition, s3ActionDefinition, ftpActionDefinition, googleDriveActionDefinition, dropboxActionDefinition, onedriveActionDefinition, manualTriggerDefinition, scheduleTriggerDefinition, webhookTriggerDefinition, httpRequestDefinition, consoleLogDefinition, delayDefinition, conditionDefinition, forEachDefinition, groupDefinition, queryDefinition, transformDefinition, debugDefinition, errorDefinition, variablesDefinition, graphqlQueryDefinition, emailDefinition, slackMessageDefinition, whatsappSendDefinition, teamsMessageDefinition, googleChatDefinition, webhookCallDefinition, aiAgentDefinition, idpExtractDefinition, };
|
|
58
|
+
export { csvActionDefinition, jsonActionDefinition, xmlActionDefinition, excelActionDefinition, s3ActionDefinition, ftpActionDefinition, googleDriveActionDefinition, dropboxActionDefinition, onedriveActionDefinition, manualTriggerDefinition, scheduleTriggerDefinition, webhookTriggerDefinition, httpRequestDefinition, consoleLogDefinition, delayDefinition, conditionDefinition, forEachDefinition, groupDefinition, queryDefinition, transformDefinition, aggregateDefinition, debugDefinition, errorDefinition, variablesDefinition, graphqlQueryDefinition, emailDefinition, slackMessageDefinition, whatsappSendDefinition, teamsMessageDefinition, googleChatDefinition, webhookCallDefinition, aiAgentDefinition, idpExtractDefinition, };
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Membership test for a readonly string tuple, narrowing the value to the
|
|
3
|
+
* tuple's union. Shared by the contracts that refuse a value outside a fixed
|
|
4
|
+
* set (Transform's mode/operation, Aggregate's function) so each names the
|
|
5
|
+
* allowed values in its message instead of failing at run time.
|
|
6
|
+
*/
|
|
7
|
+
export declare const isOneOf: <T extends string>(set: readonly T[], value: unknown) => value is T;
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The shape of ONE record at a path in an upstream step's output — what a
|
|
3
|
+
* step that reads those records sees per row — and the field names it
|
|
4
|
+
* carries. Pure and framework-free, so a panel that picks a record field
|
|
5
|
+
* (Aggregate's group-by and metric fields, DEV-422) offers the fields the
|
|
6
|
+
* Engine's `resolveArrayPath` will actually hand the step.
|
|
7
|
+
*
|
|
8
|
+
* Resolution order mirrors the ForEach panel's source resolution: the
|
|
9
|
+
* upstream node's static schema (`outputSchema`, else the editor's
|
|
10
|
+
* `config._outputSchema`) walked along the path; then the last run's sample
|
|
11
|
+
* output; then the static schema as a whole. Only the first upstream node is
|
|
12
|
+
* consulted — it is the one the step is wired from.
|
|
13
|
+
*/
|
|
14
|
+
import { type UpstreamNodeRef } from './pathOptions';
|
|
15
|
+
export interface RecordSchemaLike {
|
|
16
|
+
type?: string;
|
|
17
|
+
properties?: Record<string, any>;
|
|
18
|
+
items?: any;
|
|
19
|
+
}
|
|
20
|
+
export declare function resolveRecordSchema(upstreamNodes: UpstreamNodeRef[] | undefined, arrayPath: string): RecordSchemaLike | undefined;
|
|
21
|
+
/**
|
|
22
|
+
* The field names a record at `arrayPath` carries: the resolved schema's
|
|
23
|
+
* keys, unioned with the ACTUAL keys of the last run's records — live rows
|
|
24
|
+
* can be sparse or carry system fields a describe-derived schema omits.
|
|
25
|
+
*/
|
|
26
|
+
export declare function detectRecordFields(upstreamNodes: UpstreamNodeRef[] | undefined, arrayPath: string): string[];
|
package/dist/catalog/types.d.ts
CHANGED
package/dist/index.cjs
CHANGED
|
@@ -39,9 +39,12 @@ var __export = (target, all) => {
|
|
|
39
39
|
// src/index.ts
|
|
40
40
|
var exports_src = {};
|
|
41
41
|
__export(exports_src, {
|
|
42
|
+
AGGREGATE_FUNCTIONS: () => AGGREGATE_FUNCTIONS,
|
|
43
|
+
AGGREGATE_FUNCTION_DESCRIPTIONS: () => AGGREGATE_FUNCTION_DESCRIPTIONS,
|
|
42
44
|
ActionCategory: () => ActionCategory,
|
|
43
45
|
ActionStatus: () => ActionStatus,
|
|
44
46
|
ActionType: () => ActionType,
|
|
47
|
+
AggregateConfig: () => AggregateConfig,
|
|
45
48
|
AggregateFunction: () => AggregateFunction,
|
|
46
49
|
AiAgentConfig: () => AiAgentConfig,
|
|
47
50
|
CONDITION_OPERATORS: () => CONDITION_OPERATORS,
|
|
@@ -113,6 +116,7 @@ __export(exports_src, {
|
|
|
113
116
|
contractFor: () => contractFor,
|
|
114
117
|
contractVocabularyDrift: () => contractVocabularyDrift,
|
|
115
118
|
declaredKeys: () => declaredKeys,
|
|
119
|
+
detectRecordFields: () => detectRecordFields,
|
|
116
120
|
emailDefinition: () => emailDefinition,
|
|
117
121
|
fileEditErrors: () => fileEditErrors,
|
|
118
122
|
fileFormatErrors: () => fileFormatErrors,
|
|
@@ -130,6 +134,7 @@ __export(exports_src, {
|
|
|
130
134
|
getNodeConfigStatus: () => getNodeConfigStatus,
|
|
131
135
|
getNodeValidationStatus: () => getNodeValidationStatus,
|
|
132
136
|
getStorageProvider: () => getStorageProvider,
|
|
137
|
+
groupKeyName: () => groupKeyName,
|
|
133
138
|
inferSchemaFromSample: () => inferSchemaFromSample,
|
|
134
139
|
isEditorPrivateKey: () => isEditorPrivateKey,
|
|
135
140
|
isEventCapableConnector: () => isEventCapableConnector,
|
|
@@ -138,6 +143,7 @@ __export(exports_src, {
|
|
|
138
143
|
mappingTransformFunctionName: () => mappingTransformFunctionName,
|
|
139
144
|
migrateDelayConfig: () => migrateDelayConfig,
|
|
140
145
|
queryDefinition: () => queryDefinition,
|
|
146
|
+
resolveRecordSchema: () => resolveRecordSchema,
|
|
141
147
|
selectsWithoutValues: () => selectsWithoutValues,
|
|
142
148
|
storageConfigKeys: () => storageConfigKeys,
|
|
143
149
|
storageOperations: () => storageOperations,
|
|
@@ -202,6 +208,7 @@ var ActionType;
|
|
|
202
208
|
ActionType["SpeechToText"] = "speech_to_text";
|
|
203
209
|
ActionType["TextToSpeech"] = "text_to_speech";
|
|
204
210
|
ActionType["Transform"] = "transform";
|
|
211
|
+
ActionType["Aggregate"] = "aggregate";
|
|
205
212
|
ActionType["FileOperation"] = "file_operation";
|
|
206
213
|
ActionType["FileAction"] = "file_action";
|
|
207
214
|
ActionType["CsvAction"] = "csv_action";
|
|
@@ -1744,6 +1751,46 @@ var transformDefinition = {
|
|
|
1744
1751
|
testable: true
|
|
1745
1752
|
};
|
|
1746
1753
|
|
|
1754
|
+
// src/catalog/general/aggregate/index.ts
|
|
1755
|
+
var aggregateDefinition = {
|
|
1756
|
+
id: "aggregate",
|
|
1757
|
+
name: "Aggregate",
|
|
1758
|
+
description: "Total, count or average the upstream records into one row — or one row per group — of computed values " + "(sum, count, avg, min, max, count_distinct, first, last). Put it after the step that produces the rows and " + "before the step that writes the result, for example a TOTAL row under the rows a Transform just wrote.",
|
|
1759
|
+
category: "transform" /* Transform */,
|
|
1760
|
+
type: "aggregate" /* Aggregate */,
|
|
1761
|
+
tags: ["aggregate", "sum", "total", "count", "average", "min", "max", "group by", "rollup", "built-in"],
|
|
1762
|
+
status: "stable" /* Stable */,
|
|
1763
|
+
icon: "Sigma",
|
|
1764
|
+
color: "#0043ce",
|
|
1765
|
+
inputSchema: { fields: [] },
|
|
1766
|
+
outputSchema: {
|
|
1767
|
+
type: "object",
|
|
1768
|
+
properties: [
|
|
1769
|
+
{
|
|
1770
|
+
key: "data",
|
|
1771
|
+
type: "array" /* Array */,
|
|
1772
|
+
description: "One row per group (a single row when there is no group-by): the group keys, then each metric under its `as` key"
|
|
1773
|
+
},
|
|
1774
|
+
{
|
|
1775
|
+
key: "_meta",
|
|
1776
|
+
type: "object" /* Object */,
|
|
1777
|
+
description: "rowsIn, groups, and ignored — values skipped for a numeric metric because they were not numbers"
|
|
1778
|
+
}
|
|
1779
|
+
],
|
|
1780
|
+
sampleOutput: {
|
|
1781
|
+
data: [{ TotalAmount: 12850.4, Invoices: 35 }],
|
|
1782
|
+
_meta: { rowsIn: 35, groups: 1, ignored: 0 }
|
|
1783
|
+
},
|
|
1784
|
+
dynamic: true
|
|
1785
|
+
},
|
|
1786
|
+
version: "1.0.0",
|
|
1787
|
+
createdAt: "2026-10-08T00:00:00Z",
|
|
1788
|
+
updatedAt: "2026-10-08T00:00:00Z",
|
|
1789
|
+
createdBy: "system",
|
|
1790
|
+
isOfficial: true,
|
|
1791
|
+
testable: false
|
|
1792
|
+
};
|
|
1793
|
+
|
|
1747
1794
|
// src/catalog/storage/csv-action/index.ts
|
|
1748
1795
|
var csvActionDefinition = {
|
|
1749
1796
|
id: "csv_action",
|
|
@@ -2803,7 +2850,8 @@ var actionsRegistry = {
|
|
|
2803
2850
|
console_log: consoleLogDefinition,
|
|
2804
2851
|
delay: delayDefinition,
|
|
2805
2852
|
query: queryDefinition,
|
|
2806
|
-
transform: transformDefinition
|
|
2853
|
+
transform: transformDefinition,
|
|
2854
|
+
aggregate: aggregateDefinition
|
|
2807
2855
|
},
|
|
2808
2856
|
["logic" /* Logic */]: {
|
|
2809
2857
|
condition: conditionDefinition,
|
|
@@ -3781,6 +3829,9 @@ var EmailConfig = {
|
|
|
3781
3829
|
}
|
|
3782
3830
|
}
|
|
3783
3831
|
};
|
|
3832
|
+
// src/catalog/isOneOf.ts
|
|
3833
|
+
var isOneOf = (set, value) => typeof value === "string" && set.includes(value);
|
|
3834
|
+
|
|
3784
3835
|
// src/catalog/general/transform/config.ts
|
|
3785
3836
|
var TRANSFORM_MODES = ["visual", "expression"];
|
|
3786
3837
|
var TRANSFORM_OPERATIONS = [
|
|
@@ -3790,7 +3841,6 @@ var TRANSFORM_OPERATIONS = [
|
|
|
3790
3841
|
{ id: "delete", label: "Delete" }
|
|
3791
3842
|
];
|
|
3792
3843
|
var TRANSFORM_OPERATION_IDS = TRANSFORM_OPERATIONS.map((o) => o.id);
|
|
3793
|
-
var isOneOf = (set, value) => typeof value === "string" && set.includes(value);
|
|
3794
3844
|
var DATABASE_CONNECTOR_IDS = new Set(["mongodb", "postgresql", "mysql", "sqlite", "google_sheets"]);
|
|
3795
3845
|
var TransformConfig = {
|
|
3796
3846
|
defaults: {
|
|
@@ -3892,6 +3942,313 @@ var TransformConfig = {
|
|
|
3892
3942
|
return `map_${Date.now()}_${randomSuffix(5)}`;
|
|
3893
3943
|
}
|
|
3894
3944
|
};
|
|
3945
|
+
// src/catalog/general/aggregate/config.ts
|
|
3946
|
+
var AGGREGATE_FUNCTIONS = [
|
|
3947
|
+
"sum",
|
|
3948
|
+
"count",
|
|
3949
|
+
"avg",
|
|
3950
|
+
"min",
|
|
3951
|
+
"max",
|
|
3952
|
+
"count_distinct",
|
|
3953
|
+
"first",
|
|
3954
|
+
"last"
|
|
3955
|
+
];
|
|
3956
|
+
var AGGREGATE_FUNCTION_DESCRIPTIONS = {
|
|
3957
|
+
sum: "add up the numeric values of a field",
|
|
3958
|
+
count: "count the records (with a field: the records where it is not blank)",
|
|
3959
|
+
avg: "the mean of the numeric values of a field",
|
|
3960
|
+
min: "the smallest value of a field",
|
|
3961
|
+
max: "the largest value of a field",
|
|
3962
|
+
count_distinct: "how many different values a field has",
|
|
3963
|
+
first: "the field's value on the first record",
|
|
3964
|
+
last: "the field's value on the last record"
|
|
3965
|
+
};
|
|
3966
|
+
var RESERVED_KEYS2 = new Set(["_meta", "_lineage", "_redacted", "_error"]);
|
|
3967
|
+
var KEY_PATTERN2 = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
|
|
3968
|
+
var NUMBER_VALUED = new Set(["sum", "avg", "count", "count_distinct"]);
|
|
3969
|
+
function groupKeyName(path) {
|
|
3970
|
+
const trimmed = String(path ?? "").trim().replace(/\[\]$/, "");
|
|
3971
|
+
const segments = trimmed.split(".").filter(Boolean);
|
|
3972
|
+
return segments.length > 0 ? segments[segments.length - 1] : trimmed;
|
|
3973
|
+
}
|
|
3974
|
+
function schemaAt(schema, path) {
|
|
3975
|
+
let cur = schema;
|
|
3976
|
+
for (const part of path.replace(/\[\]$/, "").split(".").filter(Boolean)) {
|
|
3977
|
+
if (!cur)
|
|
3978
|
+
return;
|
|
3979
|
+
if (cur.type === "array" && cur.items)
|
|
3980
|
+
cur = cur.items;
|
|
3981
|
+
const next = cur.properties?.[part];
|
|
3982
|
+
if (!next)
|
|
3983
|
+
return;
|
|
3984
|
+
cur = next;
|
|
3985
|
+
}
|
|
3986
|
+
return cur;
|
|
3987
|
+
}
|
|
3988
|
+
function leafOf(node, fallback) {
|
|
3989
|
+
const t = node?.type;
|
|
3990
|
+
if (t === "number" || t === "integer" || t === "string" || t === "boolean" || t === "date")
|
|
3991
|
+
return { type: t };
|
|
3992
|
+
return { type: fallback };
|
|
3993
|
+
}
|
|
3994
|
+
function metricSchema(metric, source) {
|
|
3995
|
+
const what = metric.field ? `${metric.fn} of ${metric.field}` : metric.fn;
|
|
3996
|
+
if (NUMBER_VALUED.has(metric.fn))
|
|
3997
|
+
return { type: "number", description: what };
|
|
3998
|
+
return { ...leafOf(metric.field ? schemaAt(source, metric.field) : undefined, "string"), description: what };
|
|
3999
|
+
}
|
|
4000
|
+
var FUNCTION_LIST = AGGREGATE_FUNCTIONS.join(", ");
|
|
4001
|
+
var AggregateConfig = {
|
|
4002
|
+
schema: {
|
|
4003
|
+
arrayPath: {
|
|
4004
|
+
type: "select",
|
|
4005
|
+
required: false,
|
|
4006
|
+
dynamic: true,
|
|
4007
|
+
description: "Which upstream records to aggregate: a path to an array in the upstream step's output. " + "Leave empty for the records the previous step produced (its data). The values come from that step's data, not from the catalog."
|
|
4008
|
+
},
|
|
4009
|
+
groupBy: {
|
|
4010
|
+
type: "array",
|
|
4011
|
+
required: false,
|
|
4012
|
+
description: 'Record fields (dot-paths) to group by, e.g. ["CustomerRef.name"]. Empty: one row for all records. ' + 'Each key lands on the output row under its last segment ("name").'
|
|
4013
|
+
},
|
|
4014
|
+
metrics: {
|
|
4015
|
+
type: "array",
|
|
4016
|
+
required: true,
|
|
4017
|
+
description: 'What to compute, at least one: [{ "fn", "field", "as", "decimals"? }]. ' + `fn is one of ${FUNCTION_LIST} (` + AGGREGATE_FUNCTIONS.map((fn) => `${fn}: ${AGGREGATE_FUNCTION_DESCRIPTIONS[fn]}`).join("; ") + "). field is the record key it reads (a dot-path; count may omit it to count records). " + "as is the output key. decimals rounds a sum or avg. " + 'Example: [{ "fn": "sum", "field": "Balance", "as": "TotalAmount", "decimals": 2 }, { "fn": "count", "as": "Invoices" }]'
|
|
4018
|
+
},
|
|
4019
|
+
outputAs: {
|
|
4020
|
+
type: "string",
|
|
4021
|
+
required: false,
|
|
4022
|
+
description: "The name later steps use for this result, e.g. totals → {{totals[].TotalAmount}}. Defaults to the step name."
|
|
4023
|
+
}
|
|
4024
|
+
},
|
|
4025
|
+
defaults: {
|
|
4026
|
+
arrayPath: "",
|
|
4027
|
+
groupBy: [],
|
|
4028
|
+
metrics: [],
|
|
4029
|
+
outputAs: ""
|
|
4030
|
+
},
|
|
4031
|
+
validate(config) {
|
|
4032
|
+
const errors = [];
|
|
4033
|
+
const metrics = Array.isArray(config.metrics) ? config.metrics : [];
|
|
4034
|
+
if (metrics.length === 0) {
|
|
4035
|
+
errors.push('Add at least one metric — what to compute, e.g. { "fn": "sum", "field": "Balance", "as": "TotalAmount" }.');
|
|
4036
|
+
}
|
|
4037
|
+
const groupKeys = [];
|
|
4038
|
+
const groupBy = config.groupBy;
|
|
4039
|
+
if (groupBy !== undefined && groupBy !== null) {
|
|
4040
|
+
if (!Array.isArray(groupBy)) {
|
|
4041
|
+
errors.push('groupBy must be a list of record fields (dot-paths), e.g. ["CustomerRef.name"].');
|
|
4042
|
+
} else {
|
|
4043
|
+
for (const g of groupBy) {
|
|
4044
|
+
if (typeof g !== "string" || g.trim() === "") {
|
|
4045
|
+
errors.push("groupBy has an empty entry — each one is a record field (dot-path).");
|
|
4046
|
+
continue;
|
|
4047
|
+
}
|
|
4048
|
+
groupKeys.push(groupKeyName(g));
|
|
4049
|
+
}
|
|
4050
|
+
}
|
|
4051
|
+
}
|
|
4052
|
+
const seen = new Set;
|
|
4053
|
+
for (const key of groupKeys) {
|
|
4054
|
+
if (seen.has(key)) {
|
|
4055
|
+
errors.push(`Two group-by fields both land under "${key}" — group by fields whose last segment differs.`);
|
|
4056
|
+
}
|
|
4057
|
+
seen.add(key);
|
|
4058
|
+
}
|
|
4059
|
+
metrics.forEach((m, i) => {
|
|
4060
|
+
const where = m && typeof m === "object" && typeof m.as === "string" && m.as ? `"${m.as}"` : `metric ${i + 1}`;
|
|
4061
|
+
if (!m || typeof m !== "object") {
|
|
4062
|
+
errors.push(`${where}: a metric is an object { "fn", "field", "as" }.`);
|
|
4063
|
+
return;
|
|
4064
|
+
}
|
|
4065
|
+
const fn = m.fn;
|
|
4066
|
+
const fnValid = isOneOf(AGGREGATE_FUNCTIONS, fn);
|
|
4067
|
+
if (fn === undefined || fn === null || fn === "") {
|
|
4068
|
+
errors.push(`${where}: fn is required — one of ${FUNCTION_LIST}.`);
|
|
4069
|
+
} else if (!fnValid) {
|
|
4070
|
+
errors.push(`${where}: fn must be one of: ${FUNCTION_LIST} (got "${String(fn)}").`);
|
|
4071
|
+
}
|
|
4072
|
+
const field = m.field;
|
|
4073
|
+
if (field !== undefined && field !== null && typeof field !== "string") {
|
|
4074
|
+
errors.push(`${where}: field must be a string — a dot-path on the record, e.g. "Balance".`);
|
|
4075
|
+
} else if (fnValid && fn !== "count" && (typeof field !== "string" || field.trim() === "")) {
|
|
4076
|
+
errors.push(`${where}: ${fn} needs a field — the record key it reads, e.g. "Balance".`);
|
|
4077
|
+
}
|
|
4078
|
+
const as = m.as;
|
|
4079
|
+
if (typeof as !== "string" || as === "") {
|
|
4080
|
+
errors.push(`${where}: as is required — the output key the value lands under, e.g. "TotalAmount".`);
|
|
4081
|
+
} else if (RESERVED_KEYS2.has(as)) {
|
|
4082
|
+
errors.push(`"${as}" is reserved — pick another output key.`);
|
|
4083
|
+
} else if (!KEY_PATTERN2.test(as)) {
|
|
4084
|
+
errors.push(`"${as}" is not a valid output key (letters, digits, underscore; must not start with a digit).`);
|
|
4085
|
+
} else if (seen.has(as)) {
|
|
4086
|
+
errors.push(`Duplicate output key "${as}" — every metric and group-by key lands under its own name.`);
|
|
4087
|
+
} else {
|
|
4088
|
+
seen.add(as);
|
|
4089
|
+
}
|
|
4090
|
+
const decimals = m.decimals;
|
|
4091
|
+
if (decimals !== undefined && decimals !== null) {
|
|
4092
|
+
if (typeof decimals !== "number" || !Number.isInteger(decimals) || decimals < 0 || decimals > 10) {
|
|
4093
|
+
errors.push(`${where}: decimals must be a whole number from 0 to 10.`);
|
|
4094
|
+
}
|
|
4095
|
+
}
|
|
4096
|
+
});
|
|
4097
|
+
const outputAs = config.outputAs;
|
|
4098
|
+
if (outputAs !== undefined && outputAs !== null && outputAs !== "") {
|
|
4099
|
+
if (typeof outputAs !== "string" || !KEY_PATTERN2.test(outputAs)) {
|
|
4100
|
+
errors.push('outputAs must be a single name (letters, digits, underscore), e.g. "totals" — later steps read {{totals[].<key>}}.');
|
|
4101
|
+
}
|
|
4102
|
+
}
|
|
4103
|
+
const arrayPath = config.arrayPath;
|
|
4104
|
+
if (arrayPath !== undefined && arrayPath !== null && typeof arrayPath !== "string") {
|
|
4105
|
+
errors.push("arrayPath must be a string — a path to an array in the upstream output, or empty for its records.");
|
|
4106
|
+
}
|
|
4107
|
+
return { valid: errors.length === 0, errors };
|
|
4108
|
+
},
|
|
4109
|
+
computeOutputSchema(config, source) {
|
|
4110
|
+
const properties = {};
|
|
4111
|
+
for (const g of Array.isArray(config.groupBy) ? config.groupBy : []) {
|
|
4112
|
+
if (typeof g !== "string" || g.trim() === "")
|
|
4113
|
+
continue;
|
|
4114
|
+
properties[groupKeyName(g)] = { ...leafOf(schemaAt(source, g), "string"), description: `group: ${g}` };
|
|
4115
|
+
}
|
|
4116
|
+
for (const m of Array.isArray(config.metrics) ? config.metrics : []) {
|
|
4117
|
+
if (!m || typeof m !== "object" || typeof m.as !== "string" || !m.as || !isOneOf(AGGREGATE_FUNCTIONS, m.fn))
|
|
4118
|
+
continue;
|
|
4119
|
+
properties[m.as] = metricSchema(m, source);
|
|
4120
|
+
}
|
|
4121
|
+
return { type: "object", properties };
|
|
4122
|
+
},
|
|
4123
|
+
merge(userConfig) {
|
|
4124
|
+
return { ...AggregateConfig.defaults, ...userConfig };
|
|
4125
|
+
},
|
|
4126
|
+
getLabel(config) {
|
|
4127
|
+
const metrics = Array.isArray(config.metrics) ? config.metrics : [];
|
|
4128
|
+
const parts = metrics.filter((m) => m && typeof m === "object" && m.fn).map((m) => m.field ? `${m.fn}(${m.field})` : String(m.fn));
|
|
4129
|
+
const by = Array.isArray(config.groupBy) && config.groupBy.length > 0 ? ` by ${config.groupBy.map(groupKeyName).join(", ")}` : "";
|
|
4130
|
+
return parts.length > 0 ? `${parts.join(", ")}${by}` : "Aggregate";
|
|
4131
|
+
}
|
|
4132
|
+
};
|
|
4133
|
+
// src/catalog/logic/for-each/recordSchema.ts
|
|
4134
|
+
function resolveSchemaPath(root, path) {
|
|
4135
|
+
if (!root)
|
|
4136
|
+
return;
|
|
4137
|
+
const parts = path.split(".").filter(Boolean);
|
|
4138
|
+
let cur = root;
|
|
4139
|
+
for (const part of parts) {
|
|
4140
|
+
if (cur?.type === "object" && cur.properties && part in cur.properties) {
|
|
4141
|
+
cur = cur.properties[part];
|
|
4142
|
+
continue;
|
|
4143
|
+
}
|
|
4144
|
+
if (cur?.type === "array" && cur.items?.properties && part in cur.items.properties) {
|
|
4145
|
+
cur = cur.items.properties[part];
|
|
4146
|
+
continue;
|
|
4147
|
+
}
|
|
4148
|
+
return;
|
|
4149
|
+
}
|
|
4150
|
+
if (cur?.type === "array" && cur.items)
|
|
4151
|
+
return cur.items;
|
|
4152
|
+
if (cur?.type === "object" && cur.properties)
|
|
4153
|
+
return cur;
|
|
4154
|
+
return;
|
|
4155
|
+
}
|
|
4156
|
+
function resolveSamplePath(sample, path) {
|
|
4157
|
+
const parts = path.split(".").filter(Boolean);
|
|
4158
|
+
let cur = sample;
|
|
4159
|
+
for (const part of parts) {
|
|
4160
|
+
if (cur === null || cur === undefined)
|
|
4161
|
+
return;
|
|
4162
|
+
if (Array.isArray(cur)) {
|
|
4163
|
+
if (cur.length === 0 || typeof cur[0] !== "object" || cur[0] === null)
|
|
4164
|
+
return;
|
|
4165
|
+
cur = cur[0][part];
|
|
4166
|
+
continue;
|
|
4167
|
+
}
|
|
4168
|
+
if (typeof cur === "object") {
|
|
4169
|
+
cur = cur[part];
|
|
4170
|
+
continue;
|
|
4171
|
+
}
|
|
4172
|
+
return;
|
|
4173
|
+
}
|
|
4174
|
+
return cur;
|
|
4175
|
+
}
|
|
4176
|
+
function sampleRecords(node) {
|
|
4177
|
+
const output = node?.executionData?.output;
|
|
4178
|
+
if (!output)
|
|
4179
|
+
return null;
|
|
4180
|
+
if (Array.isArray(output))
|
|
4181
|
+
return output;
|
|
4182
|
+
const data = output.data;
|
|
4183
|
+
return Array.isArray(data) ? data : null;
|
|
4184
|
+
}
|
|
4185
|
+
var isRootPath = (path) => path === "" || path === "data";
|
|
4186
|
+
function resolveRecordSchema(upstreamNodes, arrayPath) {
|
|
4187
|
+
const node = upstreamNodes?.[0];
|
|
4188
|
+
if (!node)
|
|
4189
|
+
return;
|
|
4190
|
+
const path = (arrayPath || "").trim();
|
|
4191
|
+
const isRoot = isRootPath(path);
|
|
4192
|
+
const schema = node.outputSchema || node.config?._outputSchema;
|
|
4193
|
+
if (schema?.properties) {
|
|
4194
|
+
if (isRoot) {
|
|
4195
|
+
const hasDeepFields = Object.values(schema.properties).some((f) => f?.properties || f?.items);
|
|
4196
|
+
if (hasDeepFields)
|
|
4197
|
+
return schema;
|
|
4198
|
+
} else {
|
|
4199
|
+
const resolved = resolveSchemaPath(schema, path);
|
|
4200
|
+
if (resolved)
|
|
4201
|
+
return resolved;
|
|
4202
|
+
}
|
|
4203
|
+
}
|
|
4204
|
+
const records = sampleRecords(node);
|
|
4205
|
+
if (records && records.length > 0 && typeof records[0] === "object" && records[0] !== null) {
|
|
4206
|
+
const sample = records[0];
|
|
4207
|
+
if (isRoot)
|
|
4208
|
+
return inferSchemaFromSample(sample);
|
|
4209
|
+
const resolved = resolveSamplePath(sample, path);
|
|
4210
|
+
if (Array.isArray(resolved) && resolved.length > 0 && typeof resolved[0] === "object" && resolved[0] !== null) {
|
|
4211
|
+
return inferSchemaFromSample(resolved[0]);
|
|
4212
|
+
}
|
|
4213
|
+
if (resolved && typeof resolved === "object" && !Array.isArray(resolved)) {
|
|
4214
|
+
return inferSchemaFromSample(resolved);
|
|
4215
|
+
}
|
|
4216
|
+
if (!path.includes("."))
|
|
4217
|
+
return inferSchemaFromSample(sample);
|
|
4218
|
+
}
|
|
4219
|
+
return schema?.properties ? schema : undefined;
|
|
4220
|
+
}
|
|
4221
|
+
function detectRecordFields(upstreamNodes, arrayPath) {
|
|
4222
|
+
const set = new Set;
|
|
4223
|
+
const props = resolveRecordSchema(upstreamNodes, arrayPath)?.properties;
|
|
4224
|
+
if (props && typeof props === "object")
|
|
4225
|
+
for (const k of Object.keys(props))
|
|
4226
|
+
set.add(k);
|
|
4227
|
+
const records = sampleRecords(upstreamNodes?.[0]);
|
|
4228
|
+
if (records) {
|
|
4229
|
+
const path = (arrayPath || "").trim();
|
|
4230
|
+
const addKeys = (v) => {
|
|
4231
|
+
if (v && typeof v === "object" && !Array.isArray(v))
|
|
4232
|
+
for (const k of Object.keys(v))
|
|
4233
|
+
set.add(k);
|
|
4234
|
+
};
|
|
4235
|
+
for (const r of records.slice(0, 25)) {
|
|
4236
|
+
if (isRootPath(path)) {
|
|
4237
|
+
addKeys(r);
|
|
4238
|
+
continue;
|
|
4239
|
+
}
|
|
4240
|
+
const at = resolveSamplePath(r, path);
|
|
4241
|
+
if (Array.isArray(at))
|
|
4242
|
+
for (const item of at.slice(0, 25))
|
|
4243
|
+
addKeys(item);
|
|
4244
|
+
else if (at && typeof at === "object")
|
|
4245
|
+
addKeys(at);
|
|
4246
|
+
else if (!path.includes("."))
|
|
4247
|
+
addKeys(r);
|
|
4248
|
+
}
|
|
4249
|
+
}
|
|
4250
|
+
return Array.from(set);
|
|
4251
|
+
}
|
|
3895
4252
|
// src/catalog/storage/storageCapabilities.ts
|
|
3896
4253
|
var contentTypeField = {
|
|
3897
4254
|
key: "contentType",
|
|
@@ -5619,6 +5976,7 @@ var CONFIG_CONTRACTS = {
|
|
|
5619
5976
|
}
|
|
5620
5977
|
},
|
|
5621
5978
|
transform: { schema: TransformConfig.schema, validate: asValidator(TransformConfig.validate) },
|
|
5979
|
+
aggregate: { schema: AggregateConfig.schema, validate: asValidator(AggregateConfig.validate) },
|
|
5622
5980
|
condition: { schema: ConditionConfig.schema, validate: asValidator(ConditionConfig.validate) },
|
|
5623
5981
|
for_each: { schema: ForEachConfig.schema, validate: asValidator(ForEachConfig.validate) },
|
|
5624
5982
|
file_operation: { schema: FileOperationConfig.schema, validate: asValidator(FileOperationConfig.validate) },
|
package/dist/index.d.ts
CHANGED
|
@@ -37,6 +37,8 @@ export { queryDefinition, } from "./catalog/general/query/index";
|
|
|
37
37
|
export { isQueryableConnector, } from "./catalog/general/query/config";
|
|
38
38
|
export { AggregateFunction, type BuildQueryConfig, type ColumnSchema, type ConnectorSchema, JoinType, LogicalOperator, type QueryExecutionResult, type QueryField, type QueryFilter, type QueryFilterGroup, type QueryJoin, type QuerySort, SortDirection, type VisualQueryConfig, } from "./catalog/general/query/types";
|
|
39
39
|
export { DATABASE_CONNECTOR_IDS, type FieldMapping, TRANSFORM_MODES, TRANSFORM_OPERATIONS, TransformConfig, type TransformConfigType, type TransformOperation, } from "./catalog/general/transform/config";
|
|
40
|
+
export { AGGREGATE_FUNCTIONS, AGGREGATE_FUNCTION_DESCRIPTIONS, AggregateConfig, type AggregateConfigType, type AggregateFn, type AggregateMetric, type AggregateSchemaNode, groupKeyName, } from "./catalog/general/aggregate/config";
|
|
41
|
+
export { detectRecordFields, resolveRecordSchema, type RecordSchemaLike, } from "./catalog/logic/for-each/recordSchema";
|
|
40
42
|
export { STORAGE_CONNECTOR_IDS, type StorageExtraField, type StoragePathField, getStorageProvider, storageConfigKeys, } from "./catalog/storage/storageCapabilities";
|
|
41
43
|
export { ManualTriggerConfig } from "./catalog/triggers/manual-trigger/index";
|
|
42
44
|
export { AiAgentConfig, type AiAgentConfigType, type AiAgentOverrides } from "./catalog/ai/ai-agent/config";
|
package/dist/index.js
CHANGED
|
@@ -45,6 +45,7 @@ var ActionType;
|
|
|
45
45
|
ActionType["SpeechToText"] = "speech_to_text";
|
|
46
46
|
ActionType["TextToSpeech"] = "text_to_speech";
|
|
47
47
|
ActionType["Transform"] = "transform";
|
|
48
|
+
ActionType["Aggregate"] = "aggregate";
|
|
48
49
|
ActionType["FileOperation"] = "file_operation";
|
|
49
50
|
ActionType["FileAction"] = "file_action";
|
|
50
51
|
ActionType["CsvAction"] = "csv_action";
|
|
@@ -1587,6 +1588,46 @@ var transformDefinition = {
|
|
|
1587
1588
|
testable: true
|
|
1588
1589
|
};
|
|
1589
1590
|
|
|
1591
|
+
// src/catalog/general/aggregate/index.ts
|
|
1592
|
+
var aggregateDefinition = {
|
|
1593
|
+
id: "aggregate",
|
|
1594
|
+
name: "Aggregate",
|
|
1595
|
+
description: "Total, count or average the upstream records into one row — or one row per group — of computed values " + "(sum, count, avg, min, max, count_distinct, first, last). Put it after the step that produces the rows and " + "before the step that writes the result, for example a TOTAL row under the rows a Transform just wrote.",
|
|
1596
|
+
category: "transform" /* Transform */,
|
|
1597
|
+
type: "aggregate" /* Aggregate */,
|
|
1598
|
+
tags: ["aggregate", "sum", "total", "count", "average", "min", "max", "group by", "rollup", "built-in"],
|
|
1599
|
+
status: "stable" /* Stable */,
|
|
1600
|
+
icon: "Sigma",
|
|
1601
|
+
color: "#0043ce",
|
|
1602
|
+
inputSchema: { fields: [] },
|
|
1603
|
+
outputSchema: {
|
|
1604
|
+
type: "object",
|
|
1605
|
+
properties: [
|
|
1606
|
+
{
|
|
1607
|
+
key: "data",
|
|
1608
|
+
type: "array" /* Array */,
|
|
1609
|
+
description: "One row per group (a single row when there is no group-by): the group keys, then each metric under its `as` key"
|
|
1610
|
+
},
|
|
1611
|
+
{
|
|
1612
|
+
key: "_meta",
|
|
1613
|
+
type: "object" /* Object */,
|
|
1614
|
+
description: "rowsIn, groups, and ignored — values skipped for a numeric metric because they were not numbers"
|
|
1615
|
+
}
|
|
1616
|
+
],
|
|
1617
|
+
sampleOutput: {
|
|
1618
|
+
data: [{ TotalAmount: 12850.4, Invoices: 35 }],
|
|
1619
|
+
_meta: { rowsIn: 35, groups: 1, ignored: 0 }
|
|
1620
|
+
},
|
|
1621
|
+
dynamic: true
|
|
1622
|
+
},
|
|
1623
|
+
version: "1.0.0",
|
|
1624
|
+
createdAt: "2026-10-08T00:00:00Z",
|
|
1625
|
+
updatedAt: "2026-10-08T00:00:00Z",
|
|
1626
|
+
createdBy: "system",
|
|
1627
|
+
isOfficial: true,
|
|
1628
|
+
testable: false
|
|
1629
|
+
};
|
|
1630
|
+
|
|
1590
1631
|
// src/catalog/storage/csv-action/index.ts
|
|
1591
1632
|
var csvActionDefinition = {
|
|
1592
1633
|
id: "csv_action",
|
|
@@ -2646,7 +2687,8 @@ var actionsRegistry = {
|
|
|
2646
2687
|
console_log: consoleLogDefinition,
|
|
2647
2688
|
delay: delayDefinition,
|
|
2648
2689
|
query: queryDefinition,
|
|
2649
|
-
transform: transformDefinition
|
|
2690
|
+
transform: transformDefinition,
|
|
2691
|
+
aggregate: aggregateDefinition
|
|
2650
2692
|
},
|
|
2651
2693
|
["logic" /* Logic */]: {
|
|
2652
2694
|
condition: conditionDefinition,
|
|
@@ -3624,6 +3666,9 @@ var EmailConfig = {
|
|
|
3624
3666
|
}
|
|
3625
3667
|
}
|
|
3626
3668
|
};
|
|
3669
|
+
// src/catalog/isOneOf.ts
|
|
3670
|
+
var isOneOf = (set, value) => typeof value === "string" && set.includes(value);
|
|
3671
|
+
|
|
3627
3672
|
// src/catalog/general/transform/config.ts
|
|
3628
3673
|
var TRANSFORM_MODES = ["visual", "expression"];
|
|
3629
3674
|
var TRANSFORM_OPERATIONS = [
|
|
@@ -3633,7 +3678,6 @@ var TRANSFORM_OPERATIONS = [
|
|
|
3633
3678
|
{ id: "delete", label: "Delete" }
|
|
3634
3679
|
];
|
|
3635
3680
|
var TRANSFORM_OPERATION_IDS = TRANSFORM_OPERATIONS.map((o) => o.id);
|
|
3636
|
-
var isOneOf = (set, value) => typeof value === "string" && set.includes(value);
|
|
3637
3681
|
var DATABASE_CONNECTOR_IDS = new Set(["mongodb", "postgresql", "mysql", "sqlite", "google_sheets"]);
|
|
3638
3682
|
var TransformConfig = {
|
|
3639
3683
|
defaults: {
|
|
@@ -3735,6 +3779,313 @@ var TransformConfig = {
|
|
|
3735
3779
|
return `map_${Date.now()}_${randomSuffix(5)}`;
|
|
3736
3780
|
}
|
|
3737
3781
|
};
|
|
3782
|
+
// src/catalog/general/aggregate/config.ts
|
|
3783
|
+
var AGGREGATE_FUNCTIONS = [
|
|
3784
|
+
"sum",
|
|
3785
|
+
"count",
|
|
3786
|
+
"avg",
|
|
3787
|
+
"min",
|
|
3788
|
+
"max",
|
|
3789
|
+
"count_distinct",
|
|
3790
|
+
"first",
|
|
3791
|
+
"last"
|
|
3792
|
+
];
|
|
3793
|
+
var AGGREGATE_FUNCTION_DESCRIPTIONS = {
|
|
3794
|
+
sum: "add up the numeric values of a field",
|
|
3795
|
+
count: "count the records (with a field: the records where it is not blank)",
|
|
3796
|
+
avg: "the mean of the numeric values of a field",
|
|
3797
|
+
min: "the smallest value of a field",
|
|
3798
|
+
max: "the largest value of a field",
|
|
3799
|
+
count_distinct: "how many different values a field has",
|
|
3800
|
+
first: "the field's value on the first record",
|
|
3801
|
+
last: "the field's value on the last record"
|
|
3802
|
+
};
|
|
3803
|
+
var RESERVED_KEYS2 = new Set(["_meta", "_lineage", "_redacted", "_error"]);
|
|
3804
|
+
var KEY_PATTERN2 = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
|
|
3805
|
+
var NUMBER_VALUED = new Set(["sum", "avg", "count", "count_distinct"]);
|
|
3806
|
+
function groupKeyName(path) {
|
|
3807
|
+
const trimmed = String(path ?? "").trim().replace(/\[\]$/, "");
|
|
3808
|
+
const segments = trimmed.split(".").filter(Boolean);
|
|
3809
|
+
return segments.length > 0 ? segments[segments.length - 1] : trimmed;
|
|
3810
|
+
}
|
|
3811
|
+
function schemaAt(schema, path) {
|
|
3812
|
+
let cur = schema;
|
|
3813
|
+
for (const part of path.replace(/\[\]$/, "").split(".").filter(Boolean)) {
|
|
3814
|
+
if (!cur)
|
|
3815
|
+
return;
|
|
3816
|
+
if (cur.type === "array" && cur.items)
|
|
3817
|
+
cur = cur.items;
|
|
3818
|
+
const next = cur.properties?.[part];
|
|
3819
|
+
if (!next)
|
|
3820
|
+
return;
|
|
3821
|
+
cur = next;
|
|
3822
|
+
}
|
|
3823
|
+
return cur;
|
|
3824
|
+
}
|
|
3825
|
+
function leafOf(node, fallback) {
|
|
3826
|
+
const t = node?.type;
|
|
3827
|
+
if (t === "number" || t === "integer" || t === "string" || t === "boolean" || t === "date")
|
|
3828
|
+
return { type: t };
|
|
3829
|
+
return { type: fallback };
|
|
3830
|
+
}
|
|
3831
|
+
function metricSchema(metric, source) {
|
|
3832
|
+
const what = metric.field ? `${metric.fn} of ${metric.field}` : metric.fn;
|
|
3833
|
+
if (NUMBER_VALUED.has(metric.fn))
|
|
3834
|
+
return { type: "number", description: what };
|
|
3835
|
+
return { ...leafOf(metric.field ? schemaAt(source, metric.field) : undefined, "string"), description: what };
|
|
3836
|
+
}
|
|
3837
|
+
var FUNCTION_LIST = AGGREGATE_FUNCTIONS.join(", ");
|
|
3838
|
+
var AggregateConfig = {
|
|
3839
|
+
schema: {
|
|
3840
|
+
arrayPath: {
|
|
3841
|
+
type: "select",
|
|
3842
|
+
required: false,
|
|
3843
|
+
dynamic: true,
|
|
3844
|
+
description: "Which upstream records to aggregate: a path to an array in the upstream step's output. " + "Leave empty for the records the previous step produced (its data). The values come from that step's data, not from the catalog."
|
|
3845
|
+
},
|
|
3846
|
+
groupBy: {
|
|
3847
|
+
type: "array",
|
|
3848
|
+
required: false,
|
|
3849
|
+
description: 'Record fields (dot-paths) to group by, e.g. ["CustomerRef.name"]. Empty: one row for all records. ' + 'Each key lands on the output row under its last segment ("name").'
|
|
3850
|
+
},
|
|
3851
|
+
metrics: {
|
|
3852
|
+
type: "array",
|
|
3853
|
+
required: true,
|
|
3854
|
+
description: 'What to compute, at least one: [{ "fn", "field", "as", "decimals"? }]. ' + `fn is one of ${FUNCTION_LIST} (` + AGGREGATE_FUNCTIONS.map((fn) => `${fn}: ${AGGREGATE_FUNCTION_DESCRIPTIONS[fn]}`).join("; ") + "). field is the record key it reads (a dot-path; count may omit it to count records). " + "as is the output key. decimals rounds a sum or avg. " + 'Example: [{ "fn": "sum", "field": "Balance", "as": "TotalAmount", "decimals": 2 }, { "fn": "count", "as": "Invoices" }]'
|
|
3855
|
+
},
|
|
3856
|
+
outputAs: {
|
|
3857
|
+
type: "string",
|
|
3858
|
+
required: false,
|
|
3859
|
+
description: "The name later steps use for this result, e.g. totals → {{totals[].TotalAmount}}. Defaults to the step name."
|
|
3860
|
+
}
|
|
3861
|
+
},
|
|
3862
|
+
defaults: {
|
|
3863
|
+
arrayPath: "",
|
|
3864
|
+
groupBy: [],
|
|
3865
|
+
metrics: [],
|
|
3866
|
+
outputAs: ""
|
|
3867
|
+
},
|
|
3868
|
+
validate(config) {
|
|
3869
|
+
const errors = [];
|
|
3870
|
+
const metrics = Array.isArray(config.metrics) ? config.metrics : [];
|
|
3871
|
+
if (metrics.length === 0) {
|
|
3872
|
+
errors.push('Add at least one metric — what to compute, e.g. { "fn": "sum", "field": "Balance", "as": "TotalAmount" }.');
|
|
3873
|
+
}
|
|
3874
|
+
const groupKeys = [];
|
|
3875
|
+
const groupBy = config.groupBy;
|
|
3876
|
+
if (groupBy !== undefined && groupBy !== null) {
|
|
3877
|
+
if (!Array.isArray(groupBy)) {
|
|
3878
|
+
errors.push('groupBy must be a list of record fields (dot-paths), e.g. ["CustomerRef.name"].');
|
|
3879
|
+
} else {
|
|
3880
|
+
for (const g of groupBy) {
|
|
3881
|
+
if (typeof g !== "string" || g.trim() === "") {
|
|
3882
|
+
errors.push("groupBy has an empty entry — each one is a record field (dot-path).");
|
|
3883
|
+
continue;
|
|
3884
|
+
}
|
|
3885
|
+
groupKeys.push(groupKeyName(g));
|
|
3886
|
+
}
|
|
3887
|
+
}
|
|
3888
|
+
}
|
|
3889
|
+
const seen = new Set;
|
|
3890
|
+
for (const key of groupKeys) {
|
|
3891
|
+
if (seen.has(key)) {
|
|
3892
|
+
errors.push(`Two group-by fields both land under "${key}" — group by fields whose last segment differs.`);
|
|
3893
|
+
}
|
|
3894
|
+
seen.add(key);
|
|
3895
|
+
}
|
|
3896
|
+
metrics.forEach((m, i) => {
|
|
3897
|
+
const where = m && typeof m === "object" && typeof m.as === "string" && m.as ? `"${m.as}"` : `metric ${i + 1}`;
|
|
3898
|
+
if (!m || typeof m !== "object") {
|
|
3899
|
+
errors.push(`${where}: a metric is an object { "fn", "field", "as" }.`);
|
|
3900
|
+
return;
|
|
3901
|
+
}
|
|
3902
|
+
const fn = m.fn;
|
|
3903
|
+
const fnValid = isOneOf(AGGREGATE_FUNCTIONS, fn);
|
|
3904
|
+
if (fn === undefined || fn === null || fn === "") {
|
|
3905
|
+
errors.push(`${where}: fn is required — one of ${FUNCTION_LIST}.`);
|
|
3906
|
+
} else if (!fnValid) {
|
|
3907
|
+
errors.push(`${where}: fn must be one of: ${FUNCTION_LIST} (got "${String(fn)}").`);
|
|
3908
|
+
}
|
|
3909
|
+
const field = m.field;
|
|
3910
|
+
if (field !== undefined && field !== null && typeof field !== "string") {
|
|
3911
|
+
errors.push(`${where}: field must be a string — a dot-path on the record, e.g. "Balance".`);
|
|
3912
|
+
} else if (fnValid && fn !== "count" && (typeof field !== "string" || field.trim() === "")) {
|
|
3913
|
+
errors.push(`${where}: ${fn} needs a field — the record key it reads, e.g. "Balance".`);
|
|
3914
|
+
}
|
|
3915
|
+
const as = m.as;
|
|
3916
|
+
if (typeof as !== "string" || as === "") {
|
|
3917
|
+
errors.push(`${where}: as is required — the output key the value lands under, e.g. "TotalAmount".`);
|
|
3918
|
+
} else if (RESERVED_KEYS2.has(as)) {
|
|
3919
|
+
errors.push(`"${as}" is reserved — pick another output key.`);
|
|
3920
|
+
} else if (!KEY_PATTERN2.test(as)) {
|
|
3921
|
+
errors.push(`"${as}" is not a valid output key (letters, digits, underscore; must not start with a digit).`);
|
|
3922
|
+
} else if (seen.has(as)) {
|
|
3923
|
+
errors.push(`Duplicate output key "${as}" — every metric and group-by key lands under its own name.`);
|
|
3924
|
+
} else {
|
|
3925
|
+
seen.add(as);
|
|
3926
|
+
}
|
|
3927
|
+
const decimals = m.decimals;
|
|
3928
|
+
if (decimals !== undefined && decimals !== null) {
|
|
3929
|
+
if (typeof decimals !== "number" || !Number.isInteger(decimals) || decimals < 0 || decimals > 10) {
|
|
3930
|
+
errors.push(`${where}: decimals must be a whole number from 0 to 10.`);
|
|
3931
|
+
}
|
|
3932
|
+
}
|
|
3933
|
+
});
|
|
3934
|
+
const outputAs = config.outputAs;
|
|
3935
|
+
if (outputAs !== undefined && outputAs !== null && outputAs !== "") {
|
|
3936
|
+
if (typeof outputAs !== "string" || !KEY_PATTERN2.test(outputAs)) {
|
|
3937
|
+
errors.push('outputAs must be a single name (letters, digits, underscore), e.g. "totals" — later steps read {{totals[].<key>}}.');
|
|
3938
|
+
}
|
|
3939
|
+
}
|
|
3940
|
+
const arrayPath = config.arrayPath;
|
|
3941
|
+
if (arrayPath !== undefined && arrayPath !== null && typeof arrayPath !== "string") {
|
|
3942
|
+
errors.push("arrayPath must be a string — a path to an array in the upstream output, or empty for its records.");
|
|
3943
|
+
}
|
|
3944
|
+
return { valid: errors.length === 0, errors };
|
|
3945
|
+
},
|
|
3946
|
+
computeOutputSchema(config, source) {
|
|
3947
|
+
const properties = {};
|
|
3948
|
+
for (const g of Array.isArray(config.groupBy) ? config.groupBy : []) {
|
|
3949
|
+
if (typeof g !== "string" || g.trim() === "")
|
|
3950
|
+
continue;
|
|
3951
|
+
properties[groupKeyName(g)] = { ...leafOf(schemaAt(source, g), "string"), description: `group: ${g}` };
|
|
3952
|
+
}
|
|
3953
|
+
for (const m of Array.isArray(config.metrics) ? config.metrics : []) {
|
|
3954
|
+
if (!m || typeof m !== "object" || typeof m.as !== "string" || !m.as || !isOneOf(AGGREGATE_FUNCTIONS, m.fn))
|
|
3955
|
+
continue;
|
|
3956
|
+
properties[m.as] = metricSchema(m, source);
|
|
3957
|
+
}
|
|
3958
|
+
return { type: "object", properties };
|
|
3959
|
+
},
|
|
3960
|
+
merge(userConfig) {
|
|
3961
|
+
return { ...AggregateConfig.defaults, ...userConfig };
|
|
3962
|
+
},
|
|
3963
|
+
getLabel(config) {
|
|
3964
|
+
const metrics = Array.isArray(config.metrics) ? config.metrics : [];
|
|
3965
|
+
const parts = metrics.filter((m) => m && typeof m === "object" && m.fn).map((m) => m.field ? `${m.fn}(${m.field})` : String(m.fn));
|
|
3966
|
+
const by = Array.isArray(config.groupBy) && config.groupBy.length > 0 ? ` by ${config.groupBy.map(groupKeyName).join(", ")}` : "";
|
|
3967
|
+
return parts.length > 0 ? `${parts.join(", ")}${by}` : "Aggregate";
|
|
3968
|
+
}
|
|
3969
|
+
};
|
|
3970
|
+
// src/catalog/logic/for-each/recordSchema.ts
|
|
3971
|
+
function resolveSchemaPath(root, path) {
|
|
3972
|
+
if (!root)
|
|
3973
|
+
return;
|
|
3974
|
+
const parts = path.split(".").filter(Boolean);
|
|
3975
|
+
let cur = root;
|
|
3976
|
+
for (const part of parts) {
|
|
3977
|
+
if (cur?.type === "object" && cur.properties && part in cur.properties) {
|
|
3978
|
+
cur = cur.properties[part];
|
|
3979
|
+
continue;
|
|
3980
|
+
}
|
|
3981
|
+
if (cur?.type === "array" && cur.items?.properties && part in cur.items.properties) {
|
|
3982
|
+
cur = cur.items.properties[part];
|
|
3983
|
+
continue;
|
|
3984
|
+
}
|
|
3985
|
+
return;
|
|
3986
|
+
}
|
|
3987
|
+
if (cur?.type === "array" && cur.items)
|
|
3988
|
+
return cur.items;
|
|
3989
|
+
if (cur?.type === "object" && cur.properties)
|
|
3990
|
+
return cur;
|
|
3991
|
+
return;
|
|
3992
|
+
}
|
|
3993
|
+
function resolveSamplePath(sample, path) {
|
|
3994
|
+
const parts = path.split(".").filter(Boolean);
|
|
3995
|
+
let cur = sample;
|
|
3996
|
+
for (const part of parts) {
|
|
3997
|
+
if (cur === null || cur === undefined)
|
|
3998
|
+
return;
|
|
3999
|
+
if (Array.isArray(cur)) {
|
|
4000
|
+
if (cur.length === 0 || typeof cur[0] !== "object" || cur[0] === null)
|
|
4001
|
+
return;
|
|
4002
|
+
cur = cur[0][part];
|
|
4003
|
+
continue;
|
|
4004
|
+
}
|
|
4005
|
+
if (typeof cur === "object") {
|
|
4006
|
+
cur = cur[part];
|
|
4007
|
+
continue;
|
|
4008
|
+
}
|
|
4009
|
+
return;
|
|
4010
|
+
}
|
|
4011
|
+
return cur;
|
|
4012
|
+
}
|
|
4013
|
+
function sampleRecords(node) {
|
|
4014
|
+
const output = node?.executionData?.output;
|
|
4015
|
+
if (!output)
|
|
4016
|
+
return null;
|
|
4017
|
+
if (Array.isArray(output))
|
|
4018
|
+
return output;
|
|
4019
|
+
const data = output.data;
|
|
4020
|
+
return Array.isArray(data) ? data : null;
|
|
4021
|
+
}
|
|
4022
|
+
var isRootPath = (path) => path === "" || path === "data";
|
|
4023
|
+
function resolveRecordSchema(upstreamNodes, arrayPath) {
|
|
4024
|
+
const node = upstreamNodes?.[0];
|
|
4025
|
+
if (!node)
|
|
4026
|
+
return;
|
|
4027
|
+
const path = (arrayPath || "").trim();
|
|
4028
|
+
const isRoot = isRootPath(path);
|
|
4029
|
+
const schema = node.outputSchema || node.config?._outputSchema;
|
|
4030
|
+
if (schema?.properties) {
|
|
4031
|
+
if (isRoot) {
|
|
4032
|
+
const hasDeepFields = Object.values(schema.properties).some((f) => f?.properties || f?.items);
|
|
4033
|
+
if (hasDeepFields)
|
|
4034
|
+
return schema;
|
|
4035
|
+
} else {
|
|
4036
|
+
const resolved = resolveSchemaPath(schema, path);
|
|
4037
|
+
if (resolved)
|
|
4038
|
+
return resolved;
|
|
4039
|
+
}
|
|
4040
|
+
}
|
|
4041
|
+
const records = sampleRecords(node);
|
|
4042
|
+
if (records && records.length > 0 && typeof records[0] === "object" && records[0] !== null) {
|
|
4043
|
+
const sample = records[0];
|
|
4044
|
+
if (isRoot)
|
|
4045
|
+
return inferSchemaFromSample(sample);
|
|
4046
|
+
const resolved = resolveSamplePath(sample, path);
|
|
4047
|
+
if (Array.isArray(resolved) && resolved.length > 0 && typeof resolved[0] === "object" && resolved[0] !== null) {
|
|
4048
|
+
return inferSchemaFromSample(resolved[0]);
|
|
4049
|
+
}
|
|
4050
|
+
if (resolved && typeof resolved === "object" && !Array.isArray(resolved)) {
|
|
4051
|
+
return inferSchemaFromSample(resolved);
|
|
4052
|
+
}
|
|
4053
|
+
if (!path.includes("."))
|
|
4054
|
+
return inferSchemaFromSample(sample);
|
|
4055
|
+
}
|
|
4056
|
+
return schema?.properties ? schema : undefined;
|
|
4057
|
+
}
|
|
4058
|
+
function detectRecordFields(upstreamNodes, arrayPath) {
|
|
4059
|
+
const set = new Set;
|
|
4060
|
+
const props = resolveRecordSchema(upstreamNodes, arrayPath)?.properties;
|
|
4061
|
+
if (props && typeof props === "object")
|
|
4062
|
+
for (const k of Object.keys(props))
|
|
4063
|
+
set.add(k);
|
|
4064
|
+
const records = sampleRecords(upstreamNodes?.[0]);
|
|
4065
|
+
if (records) {
|
|
4066
|
+
const path = (arrayPath || "").trim();
|
|
4067
|
+
const addKeys = (v) => {
|
|
4068
|
+
if (v && typeof v === "object" && !Array.isArray(v))
|
|
4069
|
+
for (const k of Object.keys(v))
|
|
4070
|
+
set.add(k);
|
|
4071
|
+
};
|
|
4072
|
+
for (const r of records.slice(0, 25)) {
|
|
4073
|
+
if (isRootPath(path)) {
|
|
4074
|
+
addKeys(r);
|
|
4075
|
+
continue;
|
|
4076
|
+
}
|
|
4077
|
+
const at = resolveSamplePath(r, path);
|
|
4078
|
+
if (Array.isArray(at))
|
|
4079
|
+
for (const item of at.slice(0, 25))
|
|
4080
|
+
addKeys(item);
|
|
4081
|
+
else if (at && typeof at === "object")
|
|
4082
|
+
addKeys(at);
|
|
4083
|
+
else if (!path.includes("."))
|
|
4084
|
+
addKeys(r);
|
|
4085
|
+
}
|
|
4086
|
+
}
|
|
4087
|
+
return Array.from(set);
|
|
4088
|
+
}
|
|
3738
4089
|
// src/catalog/storage/storageCapabilities.ts
|
|
3739
4090
|
var contentTypeField = {
|
|
3740
4091
|
key: "contentType",
|
|
@@ -5462,6 +5813,7 @@ var CONFIG_CONTRACTS = {
|
|
|
5462
5813
|
}
|
|
5463
5814
|
},
|
|
5464
5815
|
transform: { schema: TransformConfig.schema, validate: asValidator(TransformConfig.validate) },
|
|
5816
|
+
aggregate: { schema: AggregateConfig.schema, validate: asValidator(AggregateConfig.validate) },
|
|
5465
5817
|
condition: { schema: ConditionConfig.schema, validate: asValidator(ConditionConfig.validate) },
|
|
5466
5818
|
for_each: { schema: ForEachConfig.schema, validate: asValidator(ForEachConfig.validate) },
|
|
5467
5819
|
file_operation: { schema: FileOperationConfig.schema, validate: asValidator(FileOperationConfig.validate) },
|
|
@@ -5813,9 +6165,12 @@ function mappingTransformFunctionName(transform) {
|
|
|
5813
6165
|
return match ? match[1] : null;
|
|
5814
6166
|
}
|
|
5815
6167
|
export {
|
|
6168
|
+
AGGREGATE_FUNCTIONS,
|
|
6169
|
+
AGGREGATE_FUNCTION_DESCRIPTIONS,
|
|
5816
6170
|
ActionCategory,
|
|
5817
6171
|
ActionStatus,
|
|
5818
6172
|
ActionType,
|
|
6173
|
+
AggregateConfig,
|
|
5819
6174
|
AggregateFunction,
|
|
5820
6175
|
AiAgentConfig,
|
|
5821
6176
|
CONDITION_OPERATORS,
|
|
@@ -5887,6 +6242,7 @@ export {
|
|
|
5887
6242
|
contractFor,
|
|
5888
6243
|
contractVocabularyDrift,
|
|
5889
6244
|
declaredKeys,
|
|
6245
|
+
detectRecordFields,
|
|
5890
6246
|
emailDefinition,
|
|
5891
6247
|
fileEditErrors,
|
|
5892
6248
|
fileFormatErrors,
|
|
@@ -5904,6 +6260,7 @@ export {
|
|
|
5904
6260
|
getNodeConfigStatus,
|
|
5905
6261
|
getNodeValidationStatus,
|
|
5906
6262
|
getStorageProvider,
|
|
6263
|
+
groupKeyName,
|
|
5907
6264
|
inferSchemaFromSample,
|
|
5908
6265
|
isEditorPrivateKey,
|
|
5909
6266
|
isEventCapableConnector,
|
|
@@ -5912,6 +6269,7 @@ export {
|
|
|
5912
6269
|
mappingTransformFunctionName,
|
|
5913
6270
|
migrateDelayConfig,
|
|
5914
6271
|
queryDefinition,
|
|
6272
|
+
resolveRecordSchema,
|
|
5915
6273
|
selectsWithoutValues,
|
|
5916
6274
|
storageConfigKeys,
|
|
5917
6275
|
storageOperations,
|
|
@@ -70,10 +70,10 @@ export interface ConfigContract {
|
|
|
70
70
|
};
|
|
71
71
|
}
|
|
72
72
|
/**
|
|
73
|
-
* Every action in the catalog, keyed by action id — since 0.11.0 all
|
|
73
|
+
* Every action in the catalog, keyed by action id — since 0.11.0 all
|
|
74
74
|
* definitions carry one (the last fifteen: group, ai_agent, idp_extract,
|
|
75
75
|
* connector_trigger, the five file-format actions and the six storage
|
|
76
|
-
* actions). `contractFor` is undefined only for an id the catalog does not
|
|
76
|
+
* actions; 0.13.0 added `aggregate`, the 38th). `contractFor` is undefined only for an id the catalog does not
|
|
77
77
|
* know, and then consumers fall back to `inputSchema` alone. A new action is
|
|
78
78
|
* a `config.ts` plus one line here; the drift test covers it from then on.
|
|
79
79
|
*/
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@octaviaflow/flow-rules",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.13.0",
|
|
4
4
|
"description": "The flow action catalog and the rules engine behind Flow Doctor — one definition of what a step accepts and what makes a flow valid, shared by the editor and the server",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.cjs",
|