@narrative.io/data-collaboration-sdk-ts 2.98.1-beta.0 → 2.100.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/collaboration-policy/types/collaboration-policy.d.ts +20 -20
- package/build/model-inference/types.d.ts +1 -1
- package/package.json +5 -5
- package/build/collaboration-policy/core/jsonschema/json-schema-types.d.ts +0 -45
- package/build/collaboration-policy/core/jsonschema/json-schema-types.js +0 -1
- package/build/collaboration-policy/core/jsonschema/policy-branches.d.ts +0 -49
- package/build/collaboration-policy/core/jsonschema/policy-branches.js +0 -1
- package/build/collaboration-policy/core/jsonschema/policy-evaluator.d.ts +0 -22
- package/build/collaboration-policy/core/jsonschema/policy-evaluator.js +0 -260
- package/build/collaboration-policy/core/jsonschema/sql-builder.d.ts +0 -79
- package/build/collaboration-policy/core/jsonschema/sql-builder.js +0 -306
- package/build/collaboration-policy/core/jsonschema/sql-poc.d.ts +0 -79
- package/build/collaboration-policy/core/jsonschema/sql-poc.js +0 -305
- package/build/nql/SubstraitParser.d.ts +0 -771
- package/build/nql/SubstraitParser.js +0 -797
|
@@ -1,260 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Collect filters for a list of required attribute names by looking them up
|
|
3
|
-
* in the property definitions object (sibling to anyOf).
|
|
4
|
-
*
|
|
5
|
-
* In the real API shape, filters live on individual property definitions:
|
|
6
|
-
*
|
|
7
|
-
* schema.properties.properties.properties = {
|
|
8
|
-
* "sha256_hashed_email": {
|
|
9
|
-
* "$ref": "#/$defs/object_value",
|
|
10
|
-
* "attribute": "https://api.narrative.io/attributes/sha256_hashed_email",
|
|
11
|
-
* "filters": [ ... ]
|
|
12
|
-
* }
|
|
13
|
-
* }
|
|
14
|
-
*
|
|
15
|
-
* @param requiredAttrs - Attribute names whose filters should be collected.
|
|
16
|
-
* @param propertyDefs - The property definitions object from the schema.
|
|
17
|
-
* @returns Collected filters from all matching property definitions.
|
|
18
|
-
*/
|
|
19
|
-
function collectFiltersForAttributes(requiredAttrs, propertyDefs) {
|
|
20
|
-
if (!propertyDefs)
|
|
21
|
-
return [];
|
|
22
|
-
const filters = [];
|
|
23
|
-
for (const attrName of requiredAttrs) {
|
|
24
|
-
const propDef = propertyDefs[attrName];
|
|
25
|
-
if (propDef?.filters && Array.isArray(propDef.filters)) {
|
|
26
|
-
filters.push(...propDef.filters);
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
|
-
return filters;
|
|
30
|
-
}
|
|
31
|
-
/**
|
|
32
|
-
* Resolve a `$ref` on a property definition to its `$defs` entry and extract
|
|
33
|
-
* the required data subfields that should generate IS NOT NULL WHERE clauses.
|
|
34
|
-
*
|
|
35
|
-
* Only object-type `$defs` (where `properties.type.const === "object"`) have
|
|
36
|
-
* data subfields. For these:
|
|
37
|
-
* - Top-level `required` entries (excluding "properties") map to data subfields (e.g., "type")
|
|
38
|
-
* - Inner `properties.properties.required` entries map to nested data subfields (e.g., "value")
|
|
39
|
-
*
|
|
40
|
-
* Primitive types (like `string_value_type`) return no subfields.
|
|
41
|
-
*
|
|
42
|
-
* @param propDef - The property definition that may have a `$ref`.
|
|
43
|
-
* @param defs - The `$defs` object from the schema root.
|
|
44
|
-
* @returns Array of subfield names that require IS NOT NULL checks.
|
|
45
|
-
*/
|
|
46
|
-
function resolveRefRequiredSubfields(propDef, defs) {
|
|
47
|
-
if (!propDef.$ref || !defs)
|
|
48
|
-
return [];
|
|
49
|
-
const defName = propDef.$ref.replace(/^#\/\$defs\//, "");
|
|
50
|
-
const def = defs[defName];
|
|
51
|
-
if (!def)
|
|
52
|
-
return [];
|
|
53
|
-
const defProperties = def.properties;
|
|
54
|
-
if (!defProperties)
|
|
55
|
-
return [];
|
|
56
|
-
// Only object-type $defs have data subfields
|
|
57
|
-
const typeProperty = defProperties.type;
|
|
58
|
-
if (typeProperty?.const !== "object")
|
|
59
|
-
return [];
|
|
60
|
-
const subfields = [];
|
|
61
|
-
// Top-level required, minus "properties" (structural wrapper)
|
|
62
|
-
const topRequired = def.required;
|
|
63
|
-
if (topRequired) {
|
|
64
|
-
for (const field of topRequired) {
|
|
65
|
-
if (field !== "properties")
|
|
66
|
-
subfields.push(field);
|
|
67
|
-
}
|
|
68
|
-
}
|
|
69
|
-
// Inner properties.properties.required
|
|
70
|
-
const innerProps = defProperties.properties;
|
|
71
|
-
const innerRequired = innerProps?.required;
|
|
72
|
-
if (innerRequired) {
|
|
73
|
-
for (const field of innerRequired) {
|
|
74
|
-
if (!subfields.includes(field))
|
|
75
|
-
subfields.push(field);
|
|
76
|
-
}
|
|
77
|
-
}
|
|
78
|
-
return subfields;
|
|
79
|
-
}
|
|
80
|
-
/**
|
|
81
|
-
* Collect required data subfields for a list of attribute names by resolving
|
|
82
|
-
* their `$ref` entries against the schema's `$defs`.
|
|
83
|
-
*
|
|
84
|
-
* These generate IS NOT NULL WHERE clauses in the SQL builder.
|
|
85
|
-
*
|
|
86
|
-
* @param requiredAttrs - Attribute names whose required subfields should be collected.
|
|
87
|
-
* @param propertyDefs - The property definitions object from the schema.
|
|
88
|
-
* @param defs - The `$defs` object from the schema root.
|
|
89
|
-
* @returns Per-attribute list of subfield paths that require IS NOT NULL checks.
|
|
90
|
-
*/
|
|
91
|
-
function collectRequiredSubfieldsFromRefs(requiredAttrs, propertyDefs, defs) {
|
|
92
|
-
if (!propertyDefs)
|
|
93
|
-
return [];
|
|
94
|
-
const result = [];
|
|
95
|
-
for (const attrName of requiredAttrs) {
|
|
96
|
-
const propDef = propertyDefs[attrName];
|
|
97
|
-
if (!propDef)
|
|
98
|
-
continue;
|
|
99
|
-
const paths = resolveRefRequiredSubfields(propDef, defs);
|
|
100
|
-
if (paths.length > 0) {
|
|
101
|
-
result.push({ attributeName: attrName, paths });
|
|
102
|
-
}
|
|
103
|
-
}
|
|
104
|
-
return result;
|
|
105
|
-
}
|
|
106
|
-
/**
|
|
107
|
-
* Determine which branches in a JSON-Schema policy are satisfied for the given attributes.
|
|
108
|
-
*
|
|
109
|
-
* Expects the real API structure:
|
|
110
|
-
*
|
|
111
|
-
* schema.properties.properties = {
|
|
112
|
-
* anyOf: [
|
|
113
|
-
* { required: ["sha256_hashed_email"] },
|
|
114
|
-
* { allOf: [
|
|
115
|
-
* { required: ["person_name"] },
|
|
116
|
-
* { required: ["iso_3166_1_country"] },
|
|
117
|
-
* { required: ["postal_address"] }
|
|
118
|
-
* ]
|
|
119
|
-
* }
|
|
120
|
-
* ],
|
|
121
|
-
* properties: {
|
|
122
|
-
* "sha256_hashed_email": {
|
|
123
|
-
* "$ref": "#/$defs/attribute_value",
|
|
124
|
-
* "attribute": "https://api.narrative.io/attributes/sha256_hashed_email",
|
|
125
|
-
* "filters": [ ... ],
|
|
126
|
-
* "$ref": "#/$defs/object_value"
|
|
127
|
-
* },
|
|
128
|
-
* ...
|
|
129
|
-
* }
|
|
130
|
-
* }
|
|
131
|
-
*
|
|
132
|
-
* Branches only contain `required` arrays. Filters are defined on the property
|
|
133
|
-
* definitions (sibling `properties` object next to `anyOf`). Required subfields
|
|
134
|
-
* for IS NOT NULL clauses are derived by resolving `$ref` entries against `$defs`.
|
|
135
|
-
*
|
|
136
|
-
* A branch is considered satisfied if:
|
|
137
|
-
* - For a simple branch: all required property names exist as Attributes by name.
|
|
138
|
-
* - For an allOf group: all sub-branches' required property names exist as Attributes.
|
|
139
|
-
*
|
|
140
|
-
* @param policy - The JSON-Schema connector policy to inspect.
|
|
141
|
-
* @param attributes - Attributes available on the dataset.
|
|
142
|
-
* @returns A list of matching branches with their required attributes, filters, and additional requirements.
|
|
143
|
-
*/
|
|
144
|
-
function findMatchingBranches(policy, attributes) {
|
|
145
|
-
const attributeByName = new Map();
|
|
146
|
-
for (const attr of attributes) {
|
|
147
|
-
attributeByName.set(attr.name, attr);
|
|
148
|
-
}
|
|
149
|
-
const schema = policy.policy;
|
|
150
|
-
const schemaProps = schema.properties;
|
|
151
|
-
// Extract $defs from the schema root for $ref resolution
|
|
152
|
-
const defs = schema.$defs;
|
|
153
|
-
// Expect something like: schema.properties.properties.anyOf
|
|
154
|
-
const propertiesObj = schemaProps &&
|
|
155
|
-
schemaProps.properties;
|
|
156
|
-
const anyOfBranches = propertiesObj?.anyOf;
|
|
157
|
-
if (!Array.isArray(anyOfBranches)) {
|
|
158
|
-
return [];
|
|
159
|
-
}
|
|
160
|
-
// Property definitions live as a sibling to anyOf — this is where filters are.
|
|
161
|
-
const propertyDefs = propertiesObj?.properties;
|
|
162
|
-
const matches = [];
|
|
163
|
-
anyOfBranches.forEach((branch, index) => {
|
|
164
|
-
if (!branch || typeof branch !== "object")
|
|
165
|
-
return;
|
|
166
|
-
const branchObj = branch;
|
|
167
|
-
// Case 1: "allOf" group branch (bundle of multiple fields)
|
|
168
|
-
if (Array.isArray(branchObj.allOf)) {
|
|
169
|
-
const groupRequiredAttrs = [];
|
|
170
|
-
let allSubBranchesSatisfied = true;
|
|
171
|
-
for (const sub of branchObj.allOf) {
|
|
172
|
-
if (!sub || typeof sub !== "object") {
|
|
173
|
-
allSubBranchesSatisfied = false;
|
|
174
|
-
break;
|
|
175
|
-
}
|
|
176
|
-
const subObj = sub;
|
|
177
|
-
const subRequired = Array.isArray(subObj.required)
|
|
178
|
-
? subObj.required
|
|
179
|
-
: [];
|
|
180
|
-
// Sub-branch is satisfied if all required keys exist as attributes.
|
|
181
|
-
const missingSub = subRequired.filter((key) => !attributeByName.has(key));
|
|
182
|
-
if (missingSub.length > 0) {
|
|
183
|
-
allSubBranchesSatisfied = false;
|
|
184
|
-
break;
|
|
185
|
-
}
|
|
186
|
-
groupRequiredAttrs.push(...subRequired);
|
|
187
|
-
}
|
|
188
|
-
if (!allSubBranchesSatisfied) {
|
|
189
|
-
return; // this outer anyOf branch is not satisfied
|
|
190
|
-
}
|
|
191
|
-
const deduped = Array.from(new Set(groupRequiredAttrs));
|
|
192
|
-
matches.push({
|
|
193
|
-
branchIndex: index,
|
|
194
|
-
requiredAttributes: deduped,
|
|
195
|
-
filters: collectFiltersForAttributes(deduped, propertyDefs),
|
|
196
|
-
additionalRequiredProperties: collectRequiredSubfieldsFromRefs(deduped, propertyDefs, defs),
|
|
197
|
-
});
|
|
198
|
-
return;
|
|
199
|
-
}
|
|
200
|
-
// Case 2: simple single-field branch
|
|
201
|
-
const branchRequired = Array.isArray(branchObj.required)
|
|
202
|
-
? branchObj.required
|
|
203
|
-
: [];
|
|
204
|
-
const missing = branchRequired.filter((key) => !attributeByName.has(key));
|
|
205
|
-
if (missing.length > 0) {
|
|
206
|
-
return; // this branch is not satisfied
|
|
207
|
-
}
|
|
208
|
-
matches.push({
|
|
209
|
-
branchIndex: index,
|
|
210
|
-
requiredAttributes: branchRequired,
|
|
211
|
-
filters: collectFiltersForAttributes(branchRequired, propertyDefs),
|
|
212
|
-
additionalRequiredProperties: collectRequiredSubfieldsFromRefs(branchRequired, propertyDefs, defs),
|
|
213
|
-
});
|
|
214
|
-
});
|
|
215
|
-
return matches;
|
|
216
|
-
}
|
|
217
|
-
/**
|
|
218
|
-
* Evaluate an array of JSON-Schema-based connector policies against a dataset.
|
|
219
|
-
*
|
|
220
|
-
* Eligibility is determined by checking whether the dataset's Rosetta Stone
|
|
221
|
-
* attributes satisfy at least one anyOf branch in the policy schema. This uses
|
|
222
|
-
* the `attributes` array (resolved from dataset mappings) as the source of truth,
|
|
223
|
-
* NOT the raw dataset.schema — the raw schema contains storage-level column names
|
|
224
|
-
* and flat types that don't match the policy's Rosetta Stone attribute names.
|
|
225
|
-
*
|
|
226
|
-
* A policy is considered valid (compatible) if `findMatchingBranches` finds at
|
|
227
|
-
* least one satisfied branch.
|
|
228
|
-
*
|
|
229
|
-
* @param _dataset - The dataset (currently unused; attributes carry the relevant info).
|
|
230
|
-
* @param attributes - Rosetta Stone attributes associated with the dataset.
|
|
231
|
-
* @param policies - JSON-Schema-based connector policies to evaluate.
|
|
232
|
-
* @returns An array of PolicyEvaluation results, one per policy.
|
|
233
|
-
*/
|
|
234
|
-
export function evaluateJsonSchemaPolicies(_dataset, attributes, policies) {
|
|
235
|
-
const evaluations = [];
|
|
236
|
-
for (const policy of policies) {
|
|
237
|
-
// Determine which branches within the policy are satisfied by the
|
|
238
|
-
// dataset's Rosetta Stone attributes.
|
|
239
|
-
const matches = findMatchingBranches(policy, attributes);
|
|
240
|
-
if (matches.length > 0) {
|
|
241
|
-
evaluations.push({
|
|
242
|
-
policy,
|
|
243
|
-
isValid: true,
|
|
244
|
-
matches,
|
|
245
|
-
errors: [],
|
|
246
|
-
});
|
|
247
|
-
}
|
|
248
|
-
else {
|
|
249
|
-
evaluations.push({
|
|
250
|
-
policy,
|
|
251
|
-
isValid: false,
|
|
252
|
-
matches: [],
|
|
253
|
-
errors: [
|
|
254
|
-
`No anyOf branches are satisfied by the available attributes for policy "${policy.name}"`,
|
|
255
|
-
],
|
|
256
|
-
});
|
|
257
|
-
}
|
|
258
|
-
}
|
|
259
|
-
return evaluations;
|
|
260
|
-
}
|
|
@@ -1,79 +0,0 @@
|
|
|
1
|
-
import type { Attribute } from "../../../attributes/types";
|
|
2
|
-
import type { Dataset } from "../../../datasets/types";
|
|
3
|
-
import type { PolicyMatchResult, PolicySqlFragments } from "../types";
|
|
4
|
-
import type { JsonSchemaConnectorPolicy } from "./json-schema-types";
|
|
5
|
-
import type { PolicyBranchMatch } from "./policy-branches";
|
|
6
|
-
/**
|
|
7
|
-
* JSON-Schema-based equivalent of PolicyMatchResult, but instead of fully-built SQL,
|
|
8
|
-
* it carries branch-level matches that your existing SQL builder can consume.
|
|
9
|
-
*/
|
|
10
|
-
export interface JsonSchemaPolicyMatchResult {
|
|
11
|
-
/** Names of policies whose JSON Schemas were compatible with the dataset. */
|
|
12
|
-
appliedPolicies: string[];
|
|
13
|
-
/** Policies that were skipped because validation failed. */
|
|
14
|
-
skippedPolicies: PolicyMatchResult["skippedPolicies"];
|
|
15
|
-
/** Non-fatal warnings (e.g. some policies were skipped). */
|
|
16
|
-
warnings: string[];
|
|
17
|
-
/**
|
|
18
|
-
* The minimum refresh schedule (most frequent) across all applied policies,
|
|
19
|
-
* expressed as an ISO 8601 duration string.
|
|
20
|
-
*/
|
|
21
|
-
refresh_schedule: string;
|
|
22
|
-
/**
|
|
23
|
-
* For each applied policy, which branches matched and what filters/attributes
|
|
24
|
-
* they require. This is what your SQL builder layer should consume.
|
|
25
|
-
*/
|
|
26
|
-
branchMatches: Array<{
|
|
27
|
-
policyName: string;
|
|
28
|
-
branches: PolicyBranchMatch[];
|
|
29
|
-
}>;
|
|
30
|
-
}
|
|
31
|
-
/**
|
|
32
|
-
* Categorize JSON-Schema-based connector policies into matching and non-matching
|
|
33
|
-
* groups based on dataset compatibility.
|
|
34
|
-
*
|
|
35
|
-
* @param policies - JSON-Schema-based connector policies.
|
|
36
|
-
* @param dataset - Dataset whose schema is used for validation.
|
|
37
|
-
* @param attributes - Attributes associated with the dataset.
|
|
38
|
-
* @returns Object with matching policies, nonMatching policies, and skip details.
|
|
39
|
-
*/
|
|
40
|
-
export declare function categorizePolicies(policies: JsonSchemaConnectorPolicy[], dataset: Dataset, attributes: Attribute[]): {
|
|
41
|
-
matching: JsonSchemaConnectorPolicy[];
|
|
42
|
-
nonMatching: JsonSchemaConnectorPolicy[];
|
|
43
|
-
skippedDetails: PolicyMatchResult["skippedPolicies"];
|
|
44
|
-
};
|
|
45
|
-
/**
|
|
46
|
-
* Helper that evaluates JSON-Schema-based policies and returns branch-level
|
|
47
|
-
* matches plus scheduling information.
|
|
48
|
-
*
|
|
49
|
-
* This is intended to be consumed by a separate SQL builder layer or for
|
|
50
|
-
* inspection of branch-level match details.
|
|
51
|
-
*
|
|
52
|
-
* @param dataset - The dataset against which policies should be evaluated.
|
|
53
|
-
* @param policies - JSON-Schema-based connector policies.
|
|
54
|
-
* @param attributes - Attributes associated with the dataset.
|
|
55
|
-
* @returns JsonSchemaPolicyMatchResult with applied/skipped policies and branch matches.
|
|
56
|
-
* @throws If no policies are compatible with the dataset.
|
|
57
|
-
*/
|
|
58
|
-
export declare function buildPolicySqlWithJsonSchema(dataset: Dataset, policies: JsonSchemaConnectorPolicy[], attributes: Attribute[]): JsonSchemaPolicyMatchResult;
|
|
59
|
-
/**
|
|
60
|
-
* Uses JSON Schema branch matching to determine valid policies, then builds
|
|
61
|
-
* SELECT and WHERE SQL fragments.
|
|
62
|
-
*
|
|
63
|
-
* @param dataset - Dataset for which to evaluate policies and produce SQL.
|
|
64
|
-
* @param policies - JSON-Schema-based connector policies.
|
|
65
|
-
* @param attributes - Attributes associated with the dataset.
|
|
66
|
-
* @returns PolicyMatchResult with select, where, applied/skipped policies, warnings, and refresh schedule.
|
|
67
|
-
*/
|
|
68
|
-
export declare function buildPolicySqlWithValidation(dataset: Dataset, policies: JsonSchemaConnectorPolicy[], attributes: Attribute[]): PolicyMatchResult;
|
|
69
|
-
/**
|
|
70
|
-
* Build SQL fragments without validation. Assumes you already filtered the policies
|
|
71
|
-
* however you want (e.g. via categorizePolicies). Builds SELECT and WHERE SQL
|
|
72
|
-
* fragments from all provided policies.
|
|
73
|
-
*
|
|
74
|
-
* @param dataset - Dataset for which to build SQL fragments.
|
|
75
|
-
* @param policies - JSON-Schema-based connector policies.
|
|
76
|
-
* @param attributes - Attributes associated with the dataset.
|
|
77
|
-
* @returns PolicySqlFragments with select and where arrays.
|
|
78
|
-
*/
|
|
79
|
-
export declare function buildPolicySql(dataset: Dataset, policies: JsonSchemaConnectorPolicy[], attributes: Attribute[]): PolicySqlFragments;
|
|
@@ -1,306 +0,0 @@
|
|
|
1
|
-
import { buildSelectAlias, resolveAttributeExpression, } from "../../utils/sql-helpers";
|
|
2
|
-
import { filterToSql } from "../filter-builder";
|
|
3
|
-
import { evaluateJsonSchemaPolicies } from "./policy-evaluator";
|
|
4
|
-
/**
|
|
5
|
-
* Convert an ISO 8601 duration string (e.g. "P1D", "PT1H") into milliseconds.
|
|
6
|
-
* This is an approximation (months and years are treated as 30 and 365 days respectively).
|
|
7
|
-
*
|
|
8
|
-
* @param duration - The ISO 8601 duration string.
|
|
9
|
-
* @returns The approximate duration in milliseconds.
|
|
10
|
-
* @throws If the duration string does not match the ISO 8601 duration format.
|
|
11
|
-
*/
|
|
12
|
-
function parseDurationToMilliseconds(duration) {
|
|
13
|
-
const regex = /^P(?:(\d+)Y)?(?:(\d+)M)?(?:(\d+)W)?(?:(\d+)D)?(?:T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?)?$/;
|
|
14
|
-
const matches = duration.match(regex);
|
|
15
|
-
if (!matches) {
|
|
16
|
-
throw new Error(`Invalid ISO 8601 duration format: ${duration}`);
|
|
17
|
-
}
|
|
18
|
-
const [, years, months, weeks, days, hours, minutes, seconds] = matches.map((m) => Number.parseInt(m || "0", 10));
|
|
19
|
-
const milliseconds = years * 365 * 24 * 60 * 60 * 1000 +
|
|
20
|
-
months * 30 * 24 * 60 * 60 * 1000 +
|
|
21
|
-
weeks * 7 * 24 * 60 * 60 * 1000 +
|
|
22
|
-
days * 24 * 60 * 60 * 1000 +
|
|
23
|
-
hours * 60 * 60 * 1000 +
|
|
24
|
-
minutes * 60 * 1000 +
|
|
25
|
-
seconds * 1000;
|
|
26
|
-
return milliseconds;
|
|
27
|
-
}
|
|
28
|
-
/**
|
|
29
|
-
* Find the "smallest" (most frequent) ISO 8601 duration within a list.
|
|
30
|
-
*
|
|
31
|
-
* @param schedules - Array of ISO 8601 duration strings.
|
|
32
|
-
* @returns The smallest duration as an ISO 8601 string, or "P1M" if none are valid.
|
|
33
|
-
*/
|
|
34
|
-
function getSmallestRefreshSchedule(schedules) {
|
|
35
|
-
const defaultMonthly = "P1M";
|
|
36
|
-
if (schedules.length === 0)
|
|
37
|
-
return defaultMonthly;
|
|
38
|
-
if (schedules.length === 1) {
|
|
39
|
-
try {
|
|
40
|
-
parseDurationToMilliseconds(schedules[0]);
|
|
41
|
-
return schedules[0];
|
|
42
|
-
}
|
|
43
|
-
catch {
|
|
44
|
-
return defaultMonthly;
|
|
45
|
-
}
|
|
46
|
-
}
|
|
47
|
-
let smallestValue = Number.MAX_SAFE_INTEGER;
|
|
48
|
-
let smallestSchedule = null;
|
|
49
|
-
for (const schedule of schedules) {
|
|
50
|
-
try {
|
|
51
|
-
const value = parseDurationToMilliseconds(schedule);
|
|
52
|
-
if (value < smallestValue) {
|
|
53
|
-
smallestValue = value;
|
|
54
|
-
smallestSchedule = schedule;
|
|
55
|
-
}
|
|
56
|
-
}
|
|
57
|
-
catch {
|
|
58
|
-
// skip invalid schedule
|
|
59
|
-
}
|
|
60
|
-
}
|
|
61
|
-
return smallestSchedule ?? defaultMonthly;
|
|
62
|
-
}
|
|
63
|
-
/**
|
|
64
|
-
* Build SQL fragments (SELECT + WHERE) from branch matches across all applied policies.
|
|
65
|
-
*
|
|
66
|
-
* SELECT: one entry per unique required attribute (root level, no subfield paths):
|
|
67
|
-
* company_data."<dataset>"."_rosetta_stone"."<attr>" AS "<attr>"
|
|
68
|
-
*
|
|
69
|
-
* WHERE: structured as root AND → per-policy structure → OR of branches.
|
|
70
|
-
* Each branch contains IS NOT NULL clauses (from additionalRequiredProperties)
|
|
71
|
-
* and filter SQL (from filters), ANDed together.
|
|
72
|
-
*
|
|
73
|
-
* @param branchMatches - Per-policy branch match results from evaluateJsonSchemaPolicies.
|
|
74
|
-
* @param attributeByName - Map of attribute name → Attribute for SQL expression building.
|
|
75
|
-
* @param datasetName - Dataset name used in SQL column paths.
|
|
76
|
-
* @returns PolicySqlFragments with select and where arrays.
|
|
77
|
-
*/
|
|
78
|
-
function buildSqlFromBranchMatches(branchMatches, attributeByName, datasetName) {
|
|
79
|
-
// --- SELECT ---
|
|
80
|
-
// Collect all unique required attributes across all policies and branches
|
|
81
|
-
const globalRequiredAttributes = new Set();
|
|
82
|
-
for (const policyMatch of branchMatches) {
|
|
83
|
-
for (const branch of policyMatch.branches) {
|
|
84
|
-
for (const attrName of branch.requiredAttributes) {
|
|
85
|
-
globalRequiredAttributes.add(attrName);
|
|
86
|
-
}
|
|
87
|
-
}
|
|
88
|
-
}
|
|
89
|
-
const selectClauses = new Set();
|
|
90
|
-
for (const attrName of globalRequiredAttributes) {
|
|
91
|
-
const attribute = attributeByName.get(attrName);
|
|
92
|
-
if (!attribute)
|
|
93
|
-
continue;
|
|
94
|
-
const expression = resolveAttributeExpression(attribute, "", datasetName);
|
|
95
|
-
const alias = buildSelectAlias(attrName, "");
|
|
96
|
-
selectClauses.add(`${expression} AS ${alias}`);
|
|
97
|
-
}
|
|
98
|
-
// --- WHERE ---
|
|
99
|
-
const policyWhereGroups = [];
|
|
100
|
-
for (const policyMatch of branchMatches) {
|
|
101
|
-
const branchWheres = [];
|
|
102
|
-
for (const branch of policyMatch.branches) {
|
|
103
|
-
const branchClauses = [];
|
|
104
|
-
// 1. IS NOT NULL clauses derived from $ref/$defs resolution
|
|
105
|
-
for (const req of branch.additionalRequiredProperties) {
|
|
106
|
-
const attribute = attributeByName.get(req.attributeName);
|
|
107
|
-
if (!attribute)
|
|
108
|
-
continue;
|
|
109
|
-
for (const path of req.paths) {
|
|
110
|
-
const expression = resolveAttributeExpression(attribute, path, datasetName);
|
|
111
|
-
branchClauses.push(`(${expression} IS NOT NULL)`);
|
|
112
|
-
}
|
|
113
|
-
}
|
|
114
|
-
// 2. Filter SQL
|
|
115
|
-
for (const filter of branch.filters) {
|
|
116
|
-
const sql = filterToSql(filter, attributeByName, datasetName);
|
|
117
|
-
branchClauses.push(sql);
|
|
118
|
-
}
|
|
119
|
-
if (branchClauses.length === 0) {
|
|
120
|
-
continue; // branch has no WHERE contribution
|
|
121
|
-
}
|
|
122
|
-
if (branchClauses.length === 1) {
|
|
123
|
-
branchWheres.push(branchClauses[0]);
|
|
124
|
-
}
|
|
125
|
-
else {
|
|
126
|
-
branchWheres.push({
|
|
127
|
-
operation: "AND",
|
|
128
|
-
fragments: branchClauses,
|
|
129
|
-
});
|
|
130
|
-
}
|
|
131
|
-
}
|
|
132
|
-
if (branchWheres.length === 0) {
|
|
133
|
-
continue; // policy has no WHERE contribution
|
|
134
|
-
}
|
|
135
|
-
if (branchWheres.length === 1) {
|
|
136
|
-
policyWhereGroups.push(branchWheres[0]);
|
|
137
|
-
}
|
|
138
|
-
else {
|
|
139
|
-
// Multiple branches → OR them (anyOf semantics)
|
|
140
|
-
policyWhereGroups.push({
|
|
141
|
-
operation: "OR",
|
|
142
|
-
fragments: branchWheres,
|
|
143
|
-
});
|
|
144
|
-
}
|
|
145
|
-
}
|
|
146
|
-
let where;
|
|
147
|
-
if (policyWhereGroups.length === 0) {
|
|
148
|
-
where = [];
|
|
149
|
-
}
|
|
150
|
-
else {
|
|
151
|
-
// Root AND wrapping all policies
|
|
152
|
-
where = [
|
|
153
|
-
{
|
|
154
|
-
fragments: policyWhereGroups,
|
|
155
|
-
operation: "AND",
|
|
156
|
-
},
|
|
157
|
-
];
|
|
158
|
-
}
|
|
159
|
-
return {
|
|
160
|
-
select: Array.from(selectClauses),
|
|
161
|
-
where,
|
|
162
|
-
};
|
|
163
|
-
}
|
|
164
|
-
/**
|
|
165
|
-
* Categorize JSON-Schema-based connector policies into matching and non-matching
|
|
166
|
-
* groups based on dataset compatibility.
|
|
167
|
-
*
|
|
168
|
-
* @param policies - JSON-Schema-based connector policies.
|
|
169
|
-
* @param dataset - Dataset whose schema is used for validation.
|
|
170
|
-
* @param attributes - Attributes associated with the dataset.
|
|
171
|
-
* @returns Object with matching policies, nonMatching policies, and skip details.
|
|
172
|
-
*/
|
|
173
|
-
export function categorizePolicies(policies, dataset, attributes) {
|
|
174
|
-
const evaluations = evaluateJsonSchemaPolicies(dataset, attributes, policies);
|
|
175
|
-
const matching = [];
|
|
176
|
-
const nonMatching = [];
|
|
177
|
-
const skippedDetails = [];
|
|
178
|
-
for (const evalResult of evaluations) {
|
|
179
|
-
const { policy, isValid, errors } = evalResult;
|
|
180
|
-
if (isValid) {
|
|
181
|
-
matching.push(policy);
|
|
182
|
-
}
|
|
183
|
-
else {
|
|
184
|
-
nonMatching.push(policy);
|
|
185
|
-
skippedDetails.push({
|
|
186
|
-
name: policy.name,
|
|
187
|
-
reason: "Dataset schema not compatible with policy JSON Schema",
|
|
188
|
-
details: errors,
|
|
189
|
-
});
|
|
190
|
-
}
|
|
191
|
-
}
|
|
192
|
-
return { matching, nonMatching, skippedDetails };
|
|
193
|
-
}
|
|
194
|
-
/**
|
|
195
|
-
* Helper that evaluates JSON-Schema-based policies and returns branch-level
|
|
196
|
-
* matches plus scheduling information.
|
|
197
|
-
*
|
|
198
|
-
* This is intended to be consumed by a separate SQL builder layer or for
|
|
199
|
-
* inspection of branch-level match details.
|
|
200
|
-
*
|
|
201
|
-
* @param dataset - The dataset against which policies should be evaluated.
|
|
202
|
-
* @param policies - JSON-Schema-based connector policies.
|
|
203
|
-
* @param attributes - Attributes associated with the dataset.
|
|
204
|
-
* @returns JsonSchemaPolicyMatchResult with applied/skipped policies and branch matches.
|
|
205
|
-
* @throws If no policies are compatible with the dataset.
|
|
206
|
-
*/
|
|
207
|
-
export function buildPolicySqlWithJsonSchema(dataset, policies, attributes) {
|
|
208
|
-
const evaluations = evaluateJsonSchemaPolicies(dataset, attributes, policies);
|
|
209
|
-
const appliedPolicies = [];
|
|
210
|
-
const skippedPolicies = [];
|
|
211
|
-
const warnings = [];
|
|
212
|
-
const branchMatches = [];
|
|
213
|
-
for (const evalResult of evaluations) {
|
|
214
|
-
const { policy, isValid, matches, errors } = evalResult;
|
|
215
|
-
if (!isValid) {
|
|
216
|
-
skippedPolicies.push({
|
|
217
|
-
name: policy.name,
|
|
218
|
-
reason: "Dataset schema not compatible with policy JSON Schema",
|
|
219
|
-
details: errors,
|
|
220
|
-
});
|
|
221
|
-
continue;
|
|
222
|
-
}
|
|
223
|
-
appliedPolicies.push(policy.name);
|
|
224
|
-
branchMatches.push({
|
|
225
|
-
policyName: policy.name,
|
|
226
|
-
branches: matches,
|
|
227
|
-
});
|
|
228
|
-
}
|
|
229
|
-
if (appliedPolicies.length === 0) {
|
|
230
|
-
const errorMessage = policies.length === 0
|
|
231
|
-
? "No policies provided"
|
|
232
|
-
: `No policies are compatible with dataset "${dataset.display_name || dataset.name}".`;
|
|
233
|
-
const details = skippedPolicies.length > 0
|
|
234
|
-
? [`Skipped policies: ${skippedPolicies.map((p) => p.name).join(", ")}`]
|
|
235
|
-
: [];
|
|
236
|
-
throw new Error(`${errorMessage}\n${details.join("\n")}`);
|
|
237
|
-
}
|
|
238
|
-
if (skippedPolicies.length > 0) {
|
|
239
|
-
warnings.push(`Skipped ${skippedPolicies.length} incompatible policies: ${skippedPolicies
|
|
240
|
-
.map((p) => p.name)
|
|
241
|
-
.join(", ")}`);
|
|
242
|
-
}
|
|
243
|
-
const refreshSchedules = evaluations
|
|
244
|
-
.filter((e) => e.isValid && e.policy.metadata.refresh_schedule?.max)
|
|
245
|
-
.map((e) => e.policy.metadata.refresh_schedule?.max)
|
|
246
|
-
.filter((max) => max != null);
|
|
247
|
-
const refresh_schedule = getSmallestRefreshSchedule(refreshSchedules);
|
|
248
|
-
return {
|
|
249
|
-
appliedPolicies,
|
|
250
|
-
skippedPolicies,
|
|
251
|
-
warnings,
|
|
252
|
-
refresh_schedule,
|
|
253
|
-
branchMatches,
|
|
254
|
-
};
|
|
255
|
-
}
|
|
256
|
-
/**
|
|
257
|
-
* Uses JSON Schema branch matching to determine valid policies, then builds
|
|
258
|
-
* SELECT and WHERE SQL fragments.
|
|
259
|
-
*
|
|
260
|
-
* @param dataset - Dataset for which to evaluate policies and produce SQL.
|
|
261
|
-
* @param policies - JSON-Schema-based connector policies.
|
|
262
|
-
* @param attributes - Attributes associated with the dataset.
|
|
263
|
-
* @returns PolicyMatchResult with select, where, applied/skipped policies, warnings, and refresh schedule.
|
|
264
|
-
*/
|
|
265
|
-
export function buildPolicySqlWithValidation(dataset, policies, attributes) {
|
|
266
|
-
const matchResult = buildPolicySqlWithJsonSchema(dataset, policies, attributes);
|
|
267
|
-
const attributeByName = new Map();
|
|
268
|
-
for (const attr of attributes) {
|
|
269
|
-
attributeByName.set(attr.name, attr);
|
|
270
|
-
}
|
|
271
|
-
const sqlFragments = buildSqlFromBranchMatches(matchResult.branchMatches, attributeByName, dataset.name);
|
|
272
|
-
return {
|
|
273
|
-
...sqlFragments,
|
|
274
|
-
appliedPolicies: matchResult.appliedPolicies,
|
|
275
|
-
skippedPolicies: matchResult.skippedPolicies,
|
|
276
|
-
warnings: matchResult.warnings,
|
|
277
|
-
refresh_schedule: matchResult.refresh_schedule,
|
|
278
|
-
};
|
|
279
|
-
}
|
|
280
|
-
/**
|
|
281
|
-
* Build SQL fragments without validation. Assumes you already filtered the policies
|
|
282
|
-
* however you want (e.g. via categorizePolicies). Builds SELECT and WHERE SQL
|
|
283
|
-
* fragments from all provided policies.
|
|
284
|
-
*
|
|
285
|
-
* @param dataset - Dataset for which to build SQL fragments.
|
|
286
|
-
* @param policies - JSON-Schema-based connector policies.
|
|
287
|
-
* @param attributes - Attributes associated with the dataset.
|
|
288
|
-
* @returns PolicySqlFragments with select and where arrays.
|
|
289
|
-
*/
|
|
290
|
-
export function buildPolicySql(dataset, policies, attributes) {
|
|
291
|
-
const evaluations = evaluateJsonSchemaPolicies(dataset, attributes, policies);
|
|
292
|
-
const attributeByName = new Map();
|
|
293
|
-
for (const attr of attributes) {
|
|
294
|
-
attributeByName.set(attr.name, attr);
|
|
295
|
-
}
|
|
296
|
-
const branchMatches = [];
|
|
297
|
-
for (const evalResult of evaluations) {
|
|
298
|
-
if (evalResult.isValid) {
|
|
299
|
-
branchMatches.push({
|
|
300
|
-
policyName: evalResult.policy.name,
|
|
301
|
-
branches: evalResult.matches,
|
|
302
|
-
});
|
|
303
|
-
}
|
|
304
|
-
}
|
|
305
|
-
return buildSqlFromBranchMatches(branchMatches, attributeByName, dataset.name);
|
|
306
|
-
}
|