@powersync/service-sync-rules 0.39.0 → 0.41.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/HydratedSyncConfig.d.ts +9 -2
- package/dist/HydratedSyncConfig.js +12 -1
- package/dist/HydratedSyncConfig.js.map +1 -1
- package/dist/HydrationState.d.ts +2 -0
- package/dist/HydrationState.js.map +1 -1
- package/dist/StorageVersion.js +2 -1
- package/dist/StorageVersion.js.map +1 -1
- package/dist/SyncConfig.d.ts +4 -3
- package/dist/SyncConfig.js +10 -11
- package/dist/SyncConfig.js.map +1 -1
- package/dist/TablePattern.d.ts +2 -0
- package/dist/TablePattern.js +18 -1
- package/dist/TablePattern.js.map +1 -1
- package/dist/compiler/bucket_resolver.d.ts +1 -0
- package/dist/compiler/bucket_resolver.js +22 -12
- package/dist/compiler/bucket_resolver.js.map +1 -1
- package/dist/compiler/compatibility.d.ts +38 -9
- package/dist/compiler/compatibility.js +81 -11
- package/dist/compiler/compatibility.js.map +1 -1
- package/dist/compiler/compiler.d.ts +81 -13
- package/dist/compiler/compiler.js +111 -15
- package/dist/compiler/compiler.js.map +1 -1
- package/dist/compiler/expression.d.ts +35 -12
- package/dist/compiler/expression.js +49 -13
- package/dist/compiler/expression.js.map +1 -1
- package/dist/compiler/filter.d.ts +4 -4
- package/dist/compiler/filter.js +7 -6
- package/dist/compiler/filter.js.map +1 -1
- package/dist/compiler/ir_to_sync_plan.d.ts +4 -2
- package/dist/compiler/ir_to_sync_plan.js +47 -37
- package/dist/compiler/ir_to_sync_plan.js.map +1 -1
- package/dist/compiler/parser.js +3 -3
- package/dist/compiler/parser.js.map +1 -1
- package/dist/compiler/querier_graph.js +45 -26
- package/dist/compiler/querier_graph.js.map +1 -1
- package/dist/compiler/rows.d.ts +23 -29
- package/dist/compiler/rows.js +83 -59
- package/dist/compiler/rows.js.map +1 -1
- package/dist/compiler/sqlite.d.ts +3 -2
- package/dist/compiler/sqlite.js +33 -3
- package/dist/compiler/sqlite.js.map +1 -1
- package/dist/compiler/table.d.ts +3 -2
- package/dist/compiler/table.js +9 -7
- package/dist/compiler/table.js.map +1 -1
- package/dist/events/CompiledEventSourceQuery.d.ts +51 -0
- package/dist/events/CompiledEventSourceQuery.js +119 -0
- package/dist/events/CompiledEventSourceQuery.js.map +1 -0
- package/dist/events/EventDescriptor.d.ts +26 -0
- package/dist/events/EventDescriptor.js +2 -0
- package/dist/events/EventDescriptor.js.map +1 -0
- package/dist/from_yaml.js +157 -189
- package/dist/from_yaml.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/dist/json_schema.js +1 -1
- package/dist/json_schema.js.map +1 -1
- package/dist/sync_plan/engine/scalar_expression_engine.js +7 -0
- package/dist/sync_plan/engine/scalar_expression_engine.js.map +1 -1
- package/dist/sync_plan/evaluator/bucket_data_source.js +11 -52
- package/dist/sync_plan/evaluator/bucket_data_source.js.map +1 -1
- package/dist/sync_plan/evaluator/index.d.ts +2 -7
- package/dist/sync_plan/evaluator/index.js +6 -7
- package/dist/sync_plan/evaluator/index.js.map +1 -1
- package/dist/sync_plan/evaluator/parameter_evaluator.js +11 -7
- package/dist/sync_plan/evaluator/parameter_evaluator.js.map +1 -1
- package/dist/sync_plan/evaluator/parameter_index_lookup_creator.js +2 -2
- package/dist/sync_plan/evaluator/parameter_index_lookup_creator.js.map +1 -1
- package/dist/sync_plan/evaluator/row_projection.d.ts +22 -0
- package/dist/sync_plan/evaluator/row_projection.js +71 -0
- package/dist/sync_plan/evaluator/row_projection.js.map +1 -0
- package/dist/sync_plan/evaluator/table_processor_to_sql.d.ts +5 -2
- package/dist/sync_plan/evaluator/table_processor_to_sql.js +10 -0
- package/dist/sync_plan/evaluator/table_processor_to_sql.js.map +1 -1
- package/dist/sync_plan/plan.d.ts +49 -9
- package/dist/sync_plan/plan_equality_serialized.d.ts +12 -2
- package/dist/sync_plan/plan_equality_serialized.js +56 -0
- package/dist/sync_plan/plan_equality_serialized.js.map +1 -1
- package/dist/sync_plan/serialize.d.ts +54 -5
- package/dist/sync_plan/serialize.js +109 -19
- package/dist/sync_plan/serialize.js.map +1 -1
- package/dist/yaml_validation.d.ts +73 -0
- package/dist/yaml_validation.js +140 -0
- package/dist/yaml_validation.js.map +1 -0
- package/package.json +1 -1
- package/schema/sync_rules.json +3 -2
- package/dist/events/SqlEventDescriptor.d.ts +0 -20
- package/dist/events/SqlEventDescriptor.js +0 -51
- package/dist/events/SqlEventDescriptor.js.map +0 -1
- package/dist/events/SqlEventSourceQuery.d.ts +0 -20
- package/dist/events/SqlEventSourceQuery.js +0 -125
- package/dist/events/SqlEventSourceQuery.js.map +0 -1
package/dist/from_yaml.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { LineCounter, parseDocument, Scalar } from 'yaml';
|
|
2
2
|
import { DEFAULT_BUCKET_PRIORITY, isValidPriority } from './BucketDescription.js';
|
|
3
3
|
import { CompatibilityContext, CompatibilityEdition, CompatibilityOption, TimeValuePrecision } from './compatibility.js';
|
|
4
4
|
import { SyncStreamsCompiler } from './compiler/compiler.js';
|
|
5
5
|
import { SqlRuleError, SyncRulesErrors, YamlError } from './errors.js';
|
|
6
|
-
import {
|
|
6
|
+
import { PreparedEventDefinition } from './events/CompiledEventSourceQuery.js';
|
|
7
7
|
import { validateSyncRulesSchema } from './json_schema.js';
|
|
8
8
|
import { SqlBucketDescriptor } from './legacy/SqlBucketDescriptor.js';
|
|
9
9
|
import { syncStreamFromSql } from './legacy/streams/from_sql.js';
|
|
@@ -12,6 +12,7 @@ import { validateStorageVersion } from './StorageVersion.js';
|
|
|
12
12
|
import { PrecompiledSyncConfig } from './sync_plan/evaluator/index.js';
|
|
13
13
|
import { TablePattern } from './TablePattern.js';
|
|
14
14
|
import { buildParsedToSourceValueMap, isBlockScalar, isQuotedScalar } from './yaml_scalar_map.js';
|
|
15
|
+
import { documentState } from './yaml_validation.js';
|
|
15
16
|
const ACCEPT_POTENTIALLY_DANGEROUS_QUERIES = Symbol('ACCEPT_POTENTIALLY_DANGEROUS_QUERIES');
|
|
16
17
|
/**
|
|
17
18
|
* Reads `sync_rules.yaml` files containing a sync configuration.
|
|
@@ -47,26 +48,44 @@ export class SyncConfigFromYaml {
|
|
|
47
48
|
}
|
|
48
49
|
]
|
|
49
50
|
});
|
|
50
|
-
|
|
51
|
+
const config = this.#parseConfig(parsed);
|
|
52
|
+
// #parseConfig() should have found all errors in the YAML source. As an additional check, and to ensure our sync
|
|
53
|
+
// rules schema is up-to-date, also validate with ajv. We do this last because errors found here don't have line
|
|
54
|
+
// numbers on them.
|
|
55
|
+
if (!this.#hasFatalError) {
|
|
56
|
+
const valid = validateSyncRulesSchema(parsed.toJSON());
|
|
57
|
+
if (!valid) {
|
|
58
|
+
this.#errors.push(...validateSyncRulesSchema.errors.map((e) => {
|
|
59
|
+
return new YamlError(e);
|
|
60
|
+
}));
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
this.#throwOnErrorIfRequested();
|
|
64
|
+
return config;
|
|
65
|
+
}
|
|
66
|
+
#parseConfig(parsed) {
|
|
67
|
+
const root = documentState(parsed, (e) => this.#errors.push(e)).requireMap('Sync Config must be a YAML map.');
|
|
68
|
+
if (parsed.errors.length > 0 || root == null) {
|
|
51
69
|
this.#errors.push(...parsed.errors.map((e) => new YamlError(e)));
|
|
52
70
|
this.#throwOnErrorIfRequested();
|
|
53
71
|
// Return an empty sync config instance if we couldn't parse YAML, it doesn't make sense to try parsing the broken
|
|
54
72
|
// structure further.
|
|
55
73
|
return new SqlSyncRules(this.yaml);
|
|
56
74
|
}
|
|
75
|
+
using rootState = root;
|
|
76
|
+
using declaredOptions = rootState.get('config')?.requireMap();
|
|
57
77
|
let compatibility;
|
|
58
78
|
let storageVersion;
|
|
59
|
-
if (
|
|
60
|
-
const declaredOptions = parsed.get('config');
|
|
79
|
+
if (declaredOptions) {
|
|
61
80
|
compatibility = this.#parseCompatibilityOptions(declaredOptions);
|
|
62
81
|
storageVersion = this.#validateStorageVersion(declaredOptions);
|
|
63
82
|
if (compatibility.isEnabled(CompatibilityOption.sqliteExpressionEngine)) {
|
|
64
83
|
// Evaluating expressions with SQLite requires edition: 3, older systems always use JavaScript.
|
|
65
|
-
if (compatibility.edition < CompatibilityEdition.
|
|
66
|
-
|
|
84
|
+
if (compatibility.edition < CompatibilityEdition.COMPILED_STREAMS) {
|
|
85
|
+
declaredOptions.reportError('Enabling unstable_sqlite_expression_engine requires edition: 3');
|
|
67
86
|
}
|
|
68
87
|
else if (!compatibility.isEnabled(CompatibilityOption.fixedJsonExtract)) {
|
|
69
|
-
|
|
88
|
+
declaredOptions.reportError('Enabling unstable_sqlite_expression_engine requires fixed_json_extract');
|
|
70
89
|
}
|
|
71
90
|
}
|
|
72
91
|
}
|
|
@@ -74,35 +93,30 @@ export class SyncConfigFromYaml {
|
|
|
74
93
|
compatibility = CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY;
|
|
75
94
|
}
|
|
76
95
|
// Bucket definitions using explicit parameter and data queries.
|
|
77
|
-
const bucketMap =
|
|
78
|
-
const streamMap =
|
|
79
|
-
const globalCtes =
|
|
96
|
+
const bucketMap = rootState.get('bucket_definitions')?.requireMap();
|
|
97
|
+
const streamMap = rootState.get('streams')?.requireMap();
|
|
98
|
+
const globalCtes = rootState.get('with')?.requireMap();
|
|
99
|
+
const eventMap = rootState.get('event_definitions')?.requireMap();
|
|
80
100
|
let result;
|
|
81
101
|
if (compatibility.edition >= CompatibilityEdition.COMPILED_STREAMS) {
|
|
82
|
-
result = this.#compileSyncPlan(bucketMap, streamMap, globalCtes, compatibility);
|
|
102
|
+
result = this.#compileSyncPlan(bucketMap, streamMap, globalCtes, eventMap, compatibility);
|
|
83
103
|
this.#warnOnUnusedCtes();
|
|
84
104
|
}
|
|
85
105
|
else {
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
106
|
+
// We don't support CTEs at all in this compiler implementation.
|
|
107
|
+
globalCtes?.reportError('Common table expressions require edition 3.');
|
|
108
|
+
const eventCompiler = new SyncStreamsCompiler({
|
|
109
|
+
defaultSchema: this.options.defaultSchema,
|
|
110
|
+
schema: this.options.schema,
|
|
111
|
+
// Editions 1 and 2 don't persist compiled plans. Keep their event behavior stable across deploys and restarts;
|
|
112
|
+
// moving to edition 3 validates and enables event payload filters.
|
|
113
|
+
compileEventPayloadFilters: false
|
|
114
|
+
});
|
|
115
|
+
this.#compileEventDefinitions(eventMap, eventCompiler);
|
|
116
|
+
const eventPlan = eventCompiler.toSyncPlan();
|
|
117
|
+
result = this.#legacyParseBucketDefinitionsAndStreams(bucketMap, streamMap, compatibility, eventPlan.events);
|
|
91
118
|
}
|
|
92
119
|
result.storageVersion = storageVersion;
|
|
93
|
-
const eventDefinitions = this.#parseEventDefinitions(parsed, compatibility);
|
|
94
|
-
result.eventDescriptors.push(...eventDefinitions);
|
|
95
|
-
// Validate that there are no additional properties.
|
|
96
|
-
// Since these errors don't contain line numbers, do this last.
|
|
97
|
-
if (!this.#hasFatalError) {
|
|
98
|
-
const valid = validateSyncRulesSchema(parsed.toJSON());
|
|
99
|
-
if (!valid) {
|
|
100
|
-
this.#errors.push(...validateSyncRulesSchema.errors.map((e) => {
|
|
101
|
-
return new YamlError(e);
|
|
102
|
-
}));
|
|
103
|
-
}
|
|
104
|
-
}
|
|
105
|
-
this.#throwOnErrorIfRequested();
|
|
106
120
|
return result;
|
|
107
121
|
}
|
|
108
122
|
get #hasFatalError() {
|
|
@@ -119,59 +133,58 @@ export class SyncConfigFromYaml {
|
|
|
119
133
|
* @see https://docs.powersync.com/sync/advanced/compatibility
|
|
120
134
|
*/
|
|
121
135
|
#parseCompatibilityOptions(declaredOptions) {
|
|
122
|
-
const edition = (declaredOptions.get('edition') ??
|
|
136
|
+
const edition = (declaredOptions.get('edition')?.requireScalar()?.requireNumeric() ??
|
|
137
|
+
CompatibilityEdition.LEGACY);
|
|
123
138
|
const options = new Map();
|
|
124
139
|
let maxTimeValuePrecision = undefined;
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
maxTimeValuePrecision = TimeValuePrecision.byName[value];
|
|
140
|
+
for (const [key, option] of Object.entries(CompatibilityOption.byName)) {
|
|
141
|
+
const isEnabled = declaredOptions.get(key)?.requireScalar()?.requireBoolean();
|
|
142
|
+
if (isEnabled != null) {
|
|
143
|
+
options.set(option, isEnabled);
|
|
130
144
|
}
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
145
|
+
}
|
|
146
|
+
const rawMaxTimestampPrecision = declaredOptions.get('timestamp_max_precision')?.requireScalar();
|
|
147
|
+
if (rawMaxTimestampPrecision != null) {
|
|
148
|
+
const stringValue = rawMaxTimestampPrecision?.requireString();
|
|
149
|
+
if (stringValue) {
|
|
150
|
+
maxTimeValuePrecision = TimeValuePrecision.byName[stringValue];
|
|
151
|
+
if (!maxTimeValuePrecision) {
|
|
152
|
+
const allowed = Object.keys(TimeValuePrecision.byName).join(', ');
|
|
153
|
+
rawMaxTimestampPrecision.reportError(`Unknown time precision, allowed are ${allowed}`);
|
|
154
|
+
}
|
|
138
155
|
}
|
|
139
156
|
}
|
|
140
157
|
const compatibility = new CompatibilityContext({ edition, overrides: options, maxTimeValuePrecision });
|
|
141
158
|
if (maxTimeValuePrecision && !compatibility.isEnabled(CompatibilityOption.timestampsIso8601)) {
|
|
142
|
-
|
|
159
|
+
declaredOptions.reportError(`'timestamp_max_precision' requires 'timestamps_iso8601' to be enabled.`);
|
|
143
160
|
}
|
|
144
161
|
return compatibility;
|
|
145
162
|
}
|
|
146
|
-
#compileSyncPlan(bucketMap, streamMap, globalCtes, compatibility) {
|
|
147
|
-
|
|
148
|
-
this.#errors.push(this.#yamlError(bucketMap, `'bucket_definitions' are not supported by the new compiler. Consider using https://powersync-community.github.io/bucket-definitions-to-sync-streams/ to translate them to streams.`));
|
|
149
|
-
}
|
|
163
|
+
#compileSyncPlan(bucketMap, streamMap, globalCtes, eventMap, compatibility) {
|
|
164
|
+
bucketMap?.reportError(`'bucket_definitions' are not supported by the new compiler. Consider using https://powersync-community.github.io/bucket-definitions-to-sync-streams/ to translate them to streams.`);
|
|
150
165
|
if (streamMap == null) {
|
|
151
166
|
this.#errors.push(new YamlError(new Error(`'streams' are required.`)));
|
|
152
167
|
}
|
|
153
168
|
const compiler = new SyncStreamsCompiler(this.options);
|
|
154
169
|
const parseCommonTableExpressions = (from) => {
|
|
155
170
|
const map = new Map();
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
const
|
|
171
|
+
const fromMap = from?.requireMap();
|
|
172
|
+
if (fromMap != null) {
|
|
173
|
+
for (const { key: cteName, keyScalar, value } of fromMap.stringKeyedItems()) {
|
|
174
|
+
const cteQuery = value.requireScalar();
|
|
160
175
|
if (this.options.schema) {
|
|
161
176
|
// Emit a warning if the CTE shadows a name from the schema.
|
|
162
177
|
const pattern = new TablePattern(this.options.defaultSchema, cteName);
|
|
163
178
|
if (this.options.schema.getTables(pattern)?.length > 0) {
|
|
164
|
-
|
|
165
|
-
error.type = 'warning';
|
|
166
|
-
this.#errors.push(error);
|
|
179
|
+
keyScalar.reportError('This common table expression shadows the name of a table in the source schema.', 'warning');
|
|
167
180
|
}
|
|
168
181
|
}
|
|
169
|
-
if (
|
|
170
|
-
const [sql, errorListener] = this.#scalarErrorListener(cteQuery);
|
|
182
|
+
if (cteQuery) {
|
|
183
|
+
const [sql, errorListener] = this.#scalarErrorListener(cteQuery.node);
|
|
171
184
|
const parsed = compiler.commonTableExpression(sql, errorListener);
|
|
172
185
|
if (parsed) {
|
|
173
186
|
const cte = { subquery: parsed, used: false };
|
|
174
|
-
this.#definedCtes.push({ cte, name:
|
|
187
|
+
this.#definedCtes.push({ cte, name: keyScalar });
|
|
175
188
|
map.set(cteName, cte);
|
|
176
189
|
}
|
|
177
190
|
}
|
|
@@ -180,19 +193,16 @@ export class SyncConfigFromYaml {
|
|
|
180
193
|
return map;
|
|
181
194
|
};
|
|
182
195
|
const parsedGlobalCommonTableExpressions = parseCommonTableExpressions(globalCtes);
|
|
183
|
-
for (const
|
|
184
|
-
const { key: keyScalar, value } = entry;
|
|
185
|
-
if (!(value instanceof YAMLMap)) {
|
|
186
|
-
// The json schema validator will flag this later.
|
|
187
|
-
continue;
|
|
188
|
-
}
|
|
189
|
-
const key = keyScalar.toString();
|
|
196
|
+
for (const { key, keyScalar, value: maybeMap } of streamMap?.stringKeyedItems() ?? []) {
|
|
190
197
|
if (!this.#checkUniqueName(key, keyScalar)) {
|
|
191
198
|
continue;
|
|
192
199
|
}
|
|
200
|
+
using value = maybeMap.requireMap();
|
|
201
|
+
if (value == null)
|
|
202
|
+
continue;
|
|
193
203
|
const streamCompiler = compiler.stream({
|
|
194
204
|
name: key,
|
|
195
|
-
isSubscribedByDefault: value.get('auto_subscribe'
|
|
205
|
+
isSubscribedByDefault: value.get('auto_subscribe')?.requireScalar()?.requireBoolean() == true,
|
|
196
206
|
priority: this.#parsePriority(value) ?? DEFAULT_BUCKET_PRIORITY,
|
|
197
207
|
warnOnDangerousParameter: !this.#acceptPotentiallyUnsafeQueries(value)
|
|
198
208
|
});
|
|
@@ -200,65 +210,58 @@ export class SyncConfigFromYaml {
|
|
|
200
210
|
// Add stream-local CTEs, which shadow global definitions.
|
|
201
211
|
parseCommonTableExpressions(value.get('with')).forEach((query, name) => streamCompiler.registerCommonTableExpression(name, query));
|
|
202
212
|
const addQuery = (query) => {
|
|
203
|
-
|
|
204
|
-
|
|
213
|
+
const scalar = query.requireScalar();
|
|
214
|
+
if (scalar) {
|
|
215
|
+
const [sql, errorListener] = this.#scalarErrorListener(scalar.node);
|
|
205
216
|
streamCompiler.addQuery(sql, errorListener);
|
|
206
217
|
}
|
|
207
218
|
};
|
|
208
219
|
const queries = value.get('queries');
|
|
209
|
-
const query = value.get('query'
|
|
220
|
+
const query = value.get('query');
|
|
210
221
|
if ((queries == null) == (query == null)) {
|
|
211
|
-
|
|
222
|
+
value.reportError('One of `queries` or `query` must be given.');
|
|
212
223
|
}
|
|
213
224
|
if (query) {
|
|
214
225
|
addQuery(query);
|
|
215
226
|
}
|
|
216
227
|
if (queries) {
|
|
217
|
-
for (const queryEntry of queries
|
|
228
|
+
for (const queryEntry of queries?.requireSequence()?.items ?? []) {
|
|
218
229
|
addQuery(queryEntry);
|
|
219
230
|
}
|
|
220
231
|
}
|
|
221
232
|
streamCompiler.finish();
|
|
222
233
|
}
|
|
223
|
-
|
|
224
|
-
return new PrecompiledSyncConfig(compiler.
|
|
234
|
+
this.#compileEventDefinitions(eventMap, compiler);
|
|
235
|
+
return new PrecompiledSyncConfig(compiler.toSyncPlan(), compatibility, {
|
|
225
236
|
defaultSchema: this.options.defaultSchema,
|
|
226
237
|
sourceText: this.yaml
|
|
227
238
|
});
|
|
228
239
|
}
|
|
229
240
|
#warnOnUnusedCtes() {
|
|
230
|
-
for (const cte of this.#definedCtes) {
|
|
231
|
-
if (!cte.
|
|
232
|
-
|
|
233
|
-
error.type = 'warning';
|
|
234
|
-
this.#errors.push(error);
|
|
241
|
+
for (const { cte, name } of this.#definedCtes) {
|
|
242
|
+
if (!cte.used) {
|
|
243
|
+
name.reportError(`This common table expression isn't referenced.`, 'warning');
|
|
235
244
|
}
|
|
236
245
|
}
|
|
237
246
|
}
|
|
238
|
-
#legacyParseBucketDefinitionsAndStreams(bucketMap, streamMap, compatibility) {
|
|
247
|
+
#legacyParseBucketDefinitionsAndStreams(bucketMap, streamMap, compatibility, events) {
|
|
239
248
|
const rules = new SqlSyncRules(this.yaml);
|
|
240
249
|
rules.compatibility = compatibility;
|
|
250
|
+
rules.eventDefinitions.push(...events.map((event) => new PreparedEventDefinition(event, this.options.defaultSchema)));
|
|
241
251
|
if (bucketMap == null && streamMap == null) {
|
|
242
252
|
this.#errors.push(new YamlError(new Error(`'bucket_definitions' or 'streams' is required`)));
|
|
243
253
|
this.#throwOnErrorIfRequested();
|
|
244
254
|
}
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
error.type = 'warning';
|
|
250
|
-
this.#errors.push(error);
|
|
251
|
-
}
|
|
252
|
-
for (let entry of bucketMap?.items ?? []) {
|
|
253
|
-
const { key: keyScalar, value } = entry;
|
|
254
|
-
const key = keyScalar.toString();
|
|
255
|
+
// This is with config.edition <= 2, we want to encourage users with streams to migrate to version 3 to use
|
|
256
|
+
// compiled sync plans.
|
|
257
|
+
streamMap?.reportError('This is using an alpha version of Sync Streams. We recommend upgrading `config.edition` to version 3 to support the latest features.', 'warning');
|
|
258
|
+
for (const { key, keyScalar, value: maybeMap } of bucketMap?.stringKeyedItems() ?? []) {
|
|
255
259
|
if (!this.#checkUniqueName(key, keyScalar)) {
|
|
256
260
|
continue;
|
|
257
261
|
}
|
|
258
|
-
|
|
259
|
-
|
|
262
|
+
using value = maybeMap.requireMap(`'${key}' bucket definition must be an object`);
|
|
263
|
+
if (value == null)
|
|
260
264
|
continue;
|
|
261
|
-
}
|
|
262
265
|
const accept_potentially_dangerous_queries = this.#acceptPotentiallyUnsafeQueries(value);
|
|
263
266
|
const parseOptionPriority = this.#parsePriority(value);
|
|
264
267
|
const queryOptions = {
|
|
@@ -267,26 +270,28 @@ export class SyncConfigFromYaml {
|
|
|
267
270
|
priority: parseOptionPriority,
|
|
268
271
|
compatibility
|
|
269
272
|
};
|
|
270
|
-
const parameters = value.get('parameters'
|
|
271
|
-
const dataQueries = value.
|
|
273
|
+
const parameters = value.get('parameters');
|
|
274
|
+
const dataQueries = value.require('data')?.requireSequence();
|
|
272
275
|
const descriptor = new SqlBucketDescriptor(key);
|
|
273
|
-
if (parameters
|
|
276
|
+
if (parameters == null) {
|
|
277
|
+
descriptor.addParameterQuery('SELECT', queryOptions);
|
|
278
|
+
}
|
|
279
|
+
else if (parameters.node instanceof Scalar) {
|
|
274
280
|
this.#withScalar(parameters, (q) => {
|
|
275
281
|
return descriptor.addParameterQuery(q, queryOptions);
|
|
276
282
|
});
|
|
277
283
|
}
|
|
278
|
-
else if (parameters instanceof YAMLSeq) {
|
|
279
|
-
for (let item of parameters.items) {
|
|
280
|
-
this.#withScalar(item, (q) => {
|
|
281
|
-
return descriptor.addParameterQuery(q, queryOptions);
|
|
282
|
-
});
|
|
283
|
-
}
|
|
284
|
-
}
|
|
285
284
|
else {
|
|
286
|
-
|
|
285
|
+
const seq = parameters.requireSequence('Parameters must be a string or array of strings');
|
|
286
|
+
if (seq != null) {
|
|
287
|
+
for (let item of seq.items) {
|
|
288
|
+
this.#withScalar(item, (q) => {
|
|
289
|
+
return descriptor.addParameterQuery(q, queryOptions);
|
|
290
|
+
});
|
|
291
|
+
}
|
|
292
|
+
}
|
|
287
293
|
}
|
|
288
|
-
if (
|
|
289
|
-
this.#errors.push(this.#tokenError((dataQueries ?? value), `'data' must be an array`));
|
|
294
|
+
if (dataQueries == null) {
|
|
290
295
|
continue;
|
|
291
296
|
}
|
|
292
297
|
for (let query of dataQueries.items) {
|
|
@@ -298,31 +303,28 @@ export class SyncConfigFromYaml {
|
|
|
298
303
|
rules.bucketDataSources.push(...descriptor.dataSources);
|
|
299
304
|
rules.bucketParameterLookupSources.push(...descriptor.parameterIndexLookupCreators);
|
|
300
305
|
}
|
|
301
|
-
for (const
|
|
302
|
-
const { key: keyScalar, value } = entry;
|
|
303
|
-
const key = keyScalar.toString();
|
|
306
|
+
for (const { key, keyScalar, value } of streamMap?.stringKeyedItems() ?? []) {
|
|
304
307
|
if (!this.#checkUniqueName(key, keyScalar)) {
|
|
305
308
|
continue;
|
|
306
309
|
}
|
|
310
|
+
using map = value.requireMap();
|
|
311
|
+
if (map == null)
|
|
312
|
+
continue;
|
|
307
313
|
// We don't support with or multiple queries in streams, those are only supported by the new compiler.
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
const
|
|
313
|
-
if (queries != null) {
|
|
314
|
-
this.#errors.push(this.#yamlError(queries, 'Multiple queries not supported without the `sync_config_compiler` option.'));
|
|
315
|
-
}
|
|
316
|
-
const accept_potentially_dangerous_queries = value.get('accept_potentially_dangerous_queries', true)?.value == true;
|
|
314
|
+
map
|
|
315
|
+
.get('with')
|
|
316
|
+
?.reportError('Common table expressions are not supported without the `sync_config_compiler` option.');
|
|
317
|
+
map.get('queries')?.reportError('Multiple queries not supported without the `sync_config_compiler` option.');
|
|
318
|
+
const accept_potentially_dangerous_queries = map.get('accept_potentially_dangerous_queries')?.requireScalar()?.requireBoolean() == true;
|
|
317
319
|
const queryOptions = {
|
|
318
320
|
...this.options,
|
|
319
321
|
accept_potentially_dangerous_queries,
|
|
320
|
-
priority: this.#parsePriority(
|
|
321
|
-
auto_subscribe:
|
|
322
|
+
priority: this.#parsePriority(map),
|
|
323
|
+
auto_subscribe: map.get('auto_subscribe')?.requireScalar()?.requireBoolean() == true,
|
|
322
324
|
compatibility
|
|
323
325
|
};
|
|
324
|
-
const data =
|
|
325
|
-
if (data
|
|
326
|
+
const data = map.get('query')?.requireScalar('Must be a string');
|
|
327
|
+
if (data != null) {
|
|
326
328
|
this.#withScalar(data, (q) => {
|
|
327
329
|
const [parsed, errors] = syncStreamFromSql(key, q, queryOptions);
|
|
328
330
|
rules.bucketSources.push(parsed);
|
|
@@ -334,112 +336,78 @@ export class SyncConfigFromYaml {
|
|
|
334
336
|
};
|
|
335
337
|
});
|
|
336
338
|
}
|
|
337
|
-
else {
|
|
338
|
-
this.#errors.push(this.#tokenError(data, 'Must be a string.'));
|
|
339
|
-
continue;
|
|
340
|
-
}
|
|
341
339
|
}
|
|
342
340
|
return rules;
|
|
343
341
|
}
|
|
344
342
|
#validateStorageVersion(config) {
|
|
345
|
-
const storageScalar = config.get('storage_version'
|
|
343
|
+
const storageScalar = config.get('storage_version')?.requireScalar();
|
|
346
344
|
if (storageScalar != null) {
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
const version = validateStorageVersion(
|
|
345
|
+
const rawVersion = storageScalar.requireNumeric('Storage version must be numeric');
|
|
346
|
+
if (rawVersion != null) {
|
|
347
|
+
const version = validateStorageVersion(rawVersion);
|
|
350
348
|
if (version == null) {
|
|
351
|
-
|
|
349
|
+
storageScalar.reportError(`Storage version ${rawVersion} is not supported`);
|
|
352
350
|
}
|
|
353
351
|
else if (!version.stable) {
|
|
354
|
-
|
|
355
|
-
error.type = 'warning';
|
|
356
|
-
this.#errors.push(error);
|
|
352
|
+
storageScalar.reportError(`Storage version ${version.version} is unstable, and may cause unexpected behavior or stop functioning in any release`, 'warning');
|
|
357
353
|
}
|
|
358
354
|
return version?.version;
|
|
359
355
|
}
|
|
360
|
-
else {
|
|
361
|
-
this.#errors.push(this.#yamlError(storageScalar, 'Storage version must be numeric'));
|
|
362
|
-
return undefined;
|
|
363
|
-
}
|
|
364
356
|
}
|
|
365
357
|
return undefined;
|
|
366
358
|
}
|
|
367
|
-
#
|
|
368
|
-
const
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
const { key, value } = event;
|
|
372
|
-
if (false == value instanceof YAMLMap) {
|
|
373
|
-
this.#errors.push(new YamlError(new Error(`Event definitions must be objects.`)));
|
|
359
|
+
#compileEventDefinitions(eventMap, compiler) {
|
|
360
|
+
for (const { key: name, value: maybeMap } of eventMap?.stringKeyedItems() ?? []) {
|
|
361
|
+
using value = maybeMap.requireMap(`Event definitions must be objects.`);
|
|
362
|
+
if (value == null)
|
|
374
363
|
continue;
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
if (false == payloads instanceof YAMLSeq) {
|
|
378
|
-
this.#errors.push(new YamlError(new Error(`Event definition payloads must be an array.`)));
|
|
364
|
+
const payloads = value.get('payloads')?.requireSequence(`Event definition payloads must be an array.`);
|
|
365
|
+
if (payloads == null)
|
|
379
366
|
continue;
|
|
380
|
-
|
|
381
|
-
const
|
|
382
|
-
|
|
383
|
-
if (
|
|
384
|
-
this.#errors.push(new YamlError(new Error(`Payload queries for events must be scalar.`)));
|
|
367
|
+
const eventCompiler = compiler.event(name);
|
|
368
|
+
for (const item of payloads.items) {
|
|
369
|
+
const itemScalar = item.requireScalar(`Payload queries for events must be scalar.`);
|
|
370
|
+
if (itemScalar == null)
|
|
385
371
|
continue;
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
return eventDescriptor.addSourceQuery(q, this.options);
|
|
389
|
-
});
|
|
372
|
+
const [sql, errorListener] = this.#scalarErrorListener(itemScalar.node);
|
|
373
|
+
eventCompiler.addSourceQuery(sql, errorListener);
|
|
390
374
|
}
|
|
391
|
-
eventDescriptors.push(eventDescriptor);
|
|
392
375
|
}
|
|
393
|
-
return eventDescriptors;
|
|
394
376
|
}
|
|
395
377
|
#checkUniqueName(name, literal) {
|
|
396
378
|
if (this.#definitionNames.has(name)) {
|
|
397
|
-
|
|
379
|
+
literal.reportError('Duplicate stream or bucket definition.');
|
|
398
380
|
return false;
|
|
399
381
|
}
|
|
400
382
|
this.#definitionNames.add(name);
|
|
401
383
|
return true;
|
|
402
384
|
}
|
|
403
385
|
#parsePriority(value) {
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
386
|
+
const message = 'Invalid priority, expected a number between 0 and 3 (inclusive).';
|
|
387
|
+
const priorityScalar = value.get('priority')?.requireScalar();
|
|
388
|
+
const rawPriority = priorityScalar?.requireNumeric(message);
|
|
389
|
+
if (priorityScalar == null || rawPriority == null)
|
|
390
|
+
return;
|
|
391
|
+
if (!isValidPriority(rawPriority)) {
|
|
392
|
+
priorityScalar.reportError(message);
|
|
393
|
+
return;
|
|
412
394
|
}
|
|
395
|
+
return rawPriority;
|
|
413
396
|
}
|
|
414
397
|
#acceptPotentiallyUnsafeQueries(definition) {
|
|
415
|
-
return definition.get('accept_potentially_dangerous_queries'
|
|
416
|
-
}
|
|
417
|
-
#yamlError(node, message) {
|
|
418
|
-
if (node.range != null) {
|
|
419
|
-
const [start, _, end] = node.range;
|
|
420
|
-
return new YamlError(new Error(message), { start, end });
|
|
421
|
-
}
|
|
422
|
-
else {
|
|
423
|
-
return new YamlError(new Error(message));
|
|
424
|
-
}
|
|
398
|
+
return definition.get('accept_potentially_dangerous_queries')?.requireScalar()?.requireBoolean() ?? false;
|
|
425
399
|
}
|
|
426
400
|
#tokenError(token, message) {
|
|
427
401
|
const start = token?.srcToken?.offset ?? 0;
|
|
428
402
|
const end = start + 1;
|
|
429
403
|
return new YamlError(new Error(message), { start, end });
|
|
430
404
|
}
|
|
431
|
-
#
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
return false;
|
|
435
|
-
}
|
|
436
|
-
return true;
|
|
437
|
-
}
|
|
438
|
-
#withScalar(scalar, cb) {
|
|
439
|
-
if (!this.#expectScalar(scalar)) {
|
|
405
|
+
#withScalar(state, cb) {
|
|
406
|
+
const scalar = state.requireScalar();
|
|
407
|
+
if (scalar == null) {
|
|
440
408
|
return;
|
|
441
409
|
}
|
|
442
|
-
const value = scalar.toString();
|
|
410
|
+
const value = scalar.node.toString();
|
|
443
411
|
const wrapped = (value) => {
|
|
444
412
|
try {
|
|
445
413
|
return cb(value);
|
|
@@ -453,7 +421,7 @@ export class SyncConfigFromYaml {
|
|
|
453
421
|
};
|
|
454
422
|
const result = wrapped(value);
|
|
455
423
|
for (let err of result.errors) {
|
|
456
|
-
this.#addErrorFromScalar(scalar, value, err);
|
|
424
|
+
this.#addErrorFromScalar(scalar.node, value, err);
|
|
457
425
|
}
|
|
458
426
|
return;
|
|
459
427
|
}
|