@narrative.io/data-collaboration-sdk-ts 2.67.0 → 2.69.0

package/build/collaboration-policy/core/sql-builder.js

@@ -1,4 +1,6 @@
+import { normalizePathValue } from "../utils/path-helpers";
 import { buildSelectAlias, resolveAttributeExpression, } from "../utils/sql-helpers";
+import { filterToSql } from "./filter-builder";
 import { traversePolicy } from "./policy-traverser";
 import { validatePolicyAgainstDataset } from "./policy-validator";
 function buildAttributeIndexes(attributes) {
@@ -10,45 +12,199 @@ function buildAttributeIndexes(attributes) {
     }
     return { byId, byName };
 }
+/**
+ * Converts an ISO 8601 duration string to milliseconds for comparison.
+ * This is an approximation for human-readable durations (months/years vary in length).
+ *
+ * @param duration - ISO 8601 duration string (e.g., "PT1H", "P1D", "P1M")
+ * @returns Duration in milliseconds
+ */
+function parseDurationToMilliseconds(duration) {
+    const regex = /^P(?:(\d+)Y)?(?:(\d+)M)?(?:(\d+)W)?(?:(\d+)D)?(?:T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?)?$/;
+    const matches = duration.match(regex);
+    if (!matches) {
+        throw new Error(`Invalid ISO 8601 duration format: ${duration}`);
+    }
+    const [, years, months, weeks, days, hours, minutes, seconds] = matches.map((m) => Number.parseInt(m || "0", 10));
+    // Approximate conversions (months = 30 days, years = 365 days)
+    const milliseconds = years * 365 * 24 * 60 * 60 * 1000 +
+        months * 30 * 24 * 60 * 60 * 1000 +
+        weeks * 7 * 24 * 60 * 60 * 1000 +
+        days * 24 * 60 * 60 * 1000 +
+        hours * 60 * 60 * 1000 +
+        minutes * 60 * 1000 +
+        seconds * 1000;
+    return milliseconds;
+}
+/**
+ * Finds the smallest (most frequent) refresh schedule from a list of ISO 8601 duration strings.
+ * The smallest duration means the most frequent refresh rate.
+ *
+ * @param schedules - Array of ISO 8601 duration strings
+ * @returns The smallest duration in milliseconds, or 2592000000 (30 days/1 month) if no schedules provided
+ */
 function getSmallestRefreshSchedule(schedules) {
+    // Default to monthly (30 days in milliseconds)
+    const defaultMonthlyMs = 30 * 24 * 60 * 60 * 1000;
     if (schedules.length === 0) {
-        return "@monthly";
-    }
-    // Define the hierarchy for refresh schedules (from smallest to largest)
-    const scheduleHierarchy = {
-        "@once": 0,
-        "@hourly": 1,
-        "@daily": 2,
-        "@weekly": 3,
-        "@monthly": 4,
-    };
-    // Find the schedule with the smallest value (most frequent)
-    let smallestSchedule = "@monthly";
-    let smallestValue = scheduleHierarchy["@monthly"];
+        return defaultMonthlyMs;
+    }
+    if (schedules.length === 1) {
+        try {
+            return parseDurationToMilliseconds(schedules[0]);
+        }
+        catch (error) {
+            console.warn(`Invalid refresh schedule: ${schedules[0]}, using default`, error);
+            return defaultMonthlyMs;
+        }
+    }
+    // Find the schedule with the smallest duration (most frequent refresh)
+    let smallestValue = Number.MAX_SAFE_INTEGER;
     for (const schedule of schedules) {
-        if (schedule in scheduleHierarchy) {
-            const value = scheduleHierarchy[schedule];
+        try {
+            const value = parseDurationToMilliseconds(schedule);
             if (value < smallestValue) {
                 smallestValue = value;
-                smallestSchedule = schedule;
             }
         }
-        else {
-            // If it's a custom cron expression, treat it as the most frequent (smallest)
-            // since we can't easily compare cron expressions
-            return schedule;
+        catch (error) {
+            // If parsing fails, skip this schedule
+            console.warn(`Skipping invalid refresh schedule: ${schedule}`, error);
         }
     }
-    return smallestSchedule;
+    // If all schedules failed to parse, return default
+    return smallestValue === Number.MAX_SAFE_INTEGER
+        ? defaultMonthlyMs
+        : smallestValue;
+}
+/**
+ * Builds structured WHERE clauses for a single branch node
+ */
+function buildBranchWhere(node, attributeByName, datasetName) {
+    if (!node || typeof node !== "object") {
+        return null;
+    }
+    // Handle field nodes (leaf nodes with attributes)
+    if ("field" in node) {
+        const extendedNode = node;
+        const field = extendedNode.field;
+        const attributeName = field.attribute_name;
+        const attribute = attributeByName.get(attributeName);
+        if (!attribute) {
+            return null;
+        }
+        const clauses = [];
+        // Add IS NOT NULL filters for additional_required_properties
+        if (field.additional_required_properties) {
+            for (const path of field.additional_required_properties) {
+                const pathStr = normalizePathValue(path);
+                const expression = `company_data."${datasetName}"."_rosetta_stone"."${attribute.name}"."${pathStr}"`;
+                clauses.push(`(${expression} IS NOT NULL)`);
+            }
+        }
+        // Add filters
+        if (extendedNode.filters) {
+            for (const filter of extendedNode.filters) {
+                const sql = filterToSql(filter, attributeByName, datasetName);
+                clauses.push(sql);
+            }
+        }
+        // If only one clause, return as string
+        if (clauses.length === 0) {
+            return null;
+        }
+        if (clauses.length === 1) {
+            return clauses[0];
+        }
+        // Multiple clauses for a single field - AND them together
+        return {
+            operation: "AND",
+            fragments: clauses,
+        };
+    }
+    // Handle anyOf nodes
+    if ("anyOf" in node && Array.isArray(node.anyOf)) {
+        const satisfiedBranches = [];
+        for (const child of node.anyOf) {
+            if (isBranchSatisfied(child, attributeByName)) {
+                const branchWhere = buildBranchWhere(child, attributeByName, datasetName);
+                if (branchWhere) {
+                    satisfiedBranches.push(branchWhere);
+                }
+            }
+        }
+        if (satisfiedBranches.length === 0) {
+            return null;
+        }
+        if (satisfiedBranches.length === 1) {
+            return satisfiedBranches[0];
+        }
+        // Multiple satisfied anyOf branches - OR them together
+        return {
+            operation: "OR",
+            fragments: satisfiedBranches,
+        };
+    }
+    // Handle allOf nodes
+    if ("allOf" in node && Array.isArray(node.allOf)) {
+        const allBranches = [];
+        for (const child of node.allOf) {
+            const branchWhere = buildBranchWhere(child, attributeByName, datasetName);
+            if (branchWhere) {
+                allBranches.push(branchWhere);
+            }
+        }
+        if (allBranches.length === 0) {
+            return null;
+        }
+        if (allBranches.length === 1) {
+            return allBranches[0];
+        }
+        // Multiple allOf branches - AND them together
+        return {
+            operation: "AND",
+            fragments: allBranches,
+        };
+    }
+    return null;
+}
+/**
+ * Checks if a branch is satisfied (all required attributes exist)
+ */
+function isBranchSatisfied(node, attributeByName) {
+    if (!node || typeof node !== "object") {
+        return true;
+    }
+    if ("field" in node) {
+        const field = node
+            .field;
+        if (field.type === "attribute") {
+            const attributeName = field.attribute_name;
+            return attributeByName.has(attributeName);
+        }
+        return true;
+    }
+    if ("anyOf" in node && Array.isArray(node.anyOf)) {
+        return node.anyOf.some((child) => isBranchSatisfied(child, attributeByName));
+    }
+    if ("allOf" in node && Array.isArray(node.allOf)) {
+        return node.allOf.every((child) => isBranchSatisfied(child, attributeByName));
+    }
+    return true;
 }
 function buildPolicySqlUsingIndexes(policies, attributeByName, datasetName) {
     const globalRequiredAttributePaths = new Map();
-    const policyWhereClauses = [];
+    const policyWhereGroups = [];
     for (const policy of policies) {
         const policyRequiredAttributePaths = new Map();
         const policyWhereClausesSet = new Set();
+        // Still use traversePolicy to collect required attributes for SELECT
         traversePolicy(policy, attributeByName, policyRequiredAttributePaths, policyWhereClausesSet, datasetName);
-        policyWhereClauses.push(Array.from(policyWhereClausesSet));
+        // Build structured WHERE for this policy
+        const policyWhere = buildBranchWhere(policy.policy.definition.structure, attributeByName, datasetName);
+        if (policyWhere) {
+            policyWhereGroups.push(policyWhere);
+        }
         for (const [attributeName, paths] of policyRequiredAttributePaths) {
             const globalPaths = globalRequiredAttributePaths.get(attributeName) ?? new Set();
             for (const path of paths) {
@@ -69,9 +225,26 @@ function buildPolicySqlUsingIndexes(policies, attributeByName, datasetName) {
         const alias = buildSelectAlias(attributeName, "");
         selectClauses.add(`${expression} AS ${alias}`);
     }
+    // Build the root AND structure
+    // Root level is always AND (combines policies)
+    // Each policy's branches are already OR'd/AND'd by buildBranchWhere
+    let where;
+    if (policyWhereGroups.length === 0) {
+        where = [];
+    }
+    else {
+        // Wrap all policies in root AND
+        // Each policy in policyWhereGroups is already a complete policy structure
+        where = [
+            {
+                fragments: policyWhereGroups,
+                operation: "AND",
+            },
+        ];
+    }
     return {
         select: Array.from(selectClauses),
-        where: policyWhereClauses,
+        where,
     };
 }
 export function buildPolicySqlWithValidation(dataset, policies, attributes) {

package/build/collaboration-policy/core/types.d.ts

@@ -7,9 +7,13 @@ export type ExtendedAttributeNode = Extract<StructureNode, {
 export type PolicyFilter = NonNullable<PolicyDefinition["filters"]>[number];
 export type PathValue = NonNullable<ExtendedAttributeNode["field"]["additional_required_properties"]>[number];
 export type AttributePathIndex = Map<string, Set<string>>;
+export interface PolicySqlGroup {
+    fragments: Array<string | PolicySqlGroup>;
+    operation: "AND" | "OR";
+}
 export interface PolicySqlFragments {
     select: string[];
-    where: string[][];
+    where: Array<PolicySqlGroup | string>;
 }
 export interface PolicyMatchResult extends PolicySqlFragments {
     appliedPolicies: string[];
@@ -19,6 +23,7 @@ export interface PolicyMatchResult extends PolicySqlFragments {
         details: string[];
     }>;
     warnings: string[];
-    refresh_schedule: string;
+    /** The minimum refresh schedule in milliseconds across all evaluated policies */
+    refresh_schedule: number;
 }
 export {};

package/build/collaboration-policy/index.d.ts

@@ -1,4 +1,4 @@
 export { buildPolicySql, buildPolicySqlWithValidation, categorizePolicies, } from "./core/sql-builder";
-export type { AttributePathIndex, ExtendedAttributeNode, PathValue, PolicyFilter, PolicyMatchResult, PolicySqlFragments, StructureNode, } from "./core/types";
+export type { AttributePathIndex, ExtendedAttributeNode, PathValue, PolicyFilter, PolicyMatchResult, PolicySqlFragments, PolicySqlGroup, StructureNode, } from "./core/types";
 export type { CollaborationPolicyInput, CollaborationPolicyType, } from "./types";
 export { buildCollaborationPolicyJsonSchema, CollaborationPolicy, } from "./types";

package/build/collaboration-policy/types/collaboration-policy.d.ts

@@ -12,13 +12,7 @@ declare const CollaborationPolicy: z.ZodObject<{
         metadata: z.ZodObject<{
             tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
             refresh_schedule: z.ZodOptional<z.ZodObject<{
-                max: z.ZodUnion<readonly [z.ZodEnum<{
-                    "@hourly": "@hourly";
-                    "@daily": "@daily";
-                    "@weekly": "@weekly";
-                    "@monthly": "@monthly";
-                    "@once": "@once";
-                }>, z.ZodString]>;
+                max: z.ZodString;
             }, z.core.$strict>>;
         }, z.core.$strip>;
         definition: z.ZodObject<{

package/build/collaboration-policy/types/collaboration-policy.js

@@ -7,15 +7,27 @@ const CollaborationPolicyIdentifier = z
     .meta({
     id: "collaboration_policy_identifier",
 });
-/** Refresh Schedule Schema */
+/**
+ * Refresh Schedule Schema
+ *
+ * Specifies how often the destination should be refreshed to ensure data doesn't expire.
+ * This is the maximum time interval allowed between destination updates. Waiting longer
+ * than this duration may result in the destination expiring data that should be retained.
+ *
+ * Uses ISO 8601 duration format (e.g., "PT1H" for 1 hour, "P1D" for 1 day).
+ * Supported values:
+ * - PT1H: Hourly (1 hour)
+ * - P1D: Daily (1 day)
+ * - P7D: Weekly (7 days)
+ * - P1M: Monthly (1 month)
+ * - P0D: Once (no recurring refresh)
+ * - Custom ISO 8601 duration strings (e.g., "PT30M" for 30 minutes, "P2W" for 2 weeks)
+ */
 const RefreshScheduleSchema = z
     .object({
-    max: z.union([
-        z.enum(["@hourly", "@daily", "@weekly", "@monthly", "@once"]),
-        z
-            .string()
-            .regex(/^([0-5]?[0-9]|\*) ([01]?[0-9]|2[0-3]|\*) ([0-2]?[0-9]|3[01]|\*) ([0]?[1-9]|1[0-2]|\*) ([0-6]|\*)$/, "Invalid cron expression"),
-    ]),
+    max: z
+        .string()
+        .regex(/^P(?!$)(\d+Y)?(\d+M)?(\d+W)?(\d+D)?(T(?=\d)(\d+H)?(\d+M)?(\d+S)?)?$/, "Invalid ISO 8601 duration format. Expected format like 'PT1H' (1 hour), 'P1D' (1 day), 'P7D' (7 days), 'P1M' (1 month), or 'P0D' (once)"),
 })
     .strict()
     .meta({

package/build/collaboration-policy/utils/sql-helpers.d.ts

@@ -1,5 +1,5 @@
 import type { Attribute } from "../../attributes/types";
+import type { PolicySqlGroup } from "../core/types";
 export declare function resolveAttributeExpression(attribute: Attribute, path: string, datasetName: string): string;
 export declare function buildSelectAlias(attributeName: string, path: string): string;
-export declare function buildPolicyWhereGroups(policyWhereClauses: string[][]): string[];
-export declare function formatWhereClause(policyWhereClauses: string[][]): string;
+export declare function formatWhereClause(where: Array<PolicySqlGroup | string>): string;

package/build/collaboration-policy/utils/sql-helpers.js

@@ -35,29 +35,35 @@ export function buildSelectAlias(attributeName, path) {
     const suffix = path.replace(/[^a-zA-Z0-9]+/g, "_");
     return quoteIdentifier(`${attributeName}_${suffix}`);
 }
-export function buildPolicyWhereGroups(policyWhereClauses) {
-    const groups = [];
-    for (const clauses of policyWhereClauses) {
-        if (clauses.length === 0) {
-            continue;
-        }
-        const firstClause = clauses[0];
-        if (firstClause === undefined) {
-            continue;
-        }
-        groups.push(clauses.length === 1 ? firstClause : `(${clauses.join(" AND ")})`);
+/**
+ * Recursively builds SQL string from PolicySqlGroup structure
+ */
+function buildSqlFromGroup(item) {
+    if (typeof item === "string") {
+        return item;
     }
-    return groups;
+    const { fragments, operation } = item;
+    const separator = operation === "AND" ? " AND " : " OR ";
+    const parts = fragments.map(buildSqlFromGroup);
+    if (parts.length === 0) {
+        return "";
+    }
+    if (parts.length === 1) {
+        return parts[0];
+    }
+    return `(${parts.join(separator)})`;
 }
-export function formatWhereClause(policyWhereClauses) {
-    const groups = buildPolicyWhereGroups(policyWhereClauses);
-    if (groups.length === 0) {
+export function formatWhereClause(where) {
+    if (where.length === 0) {
         return "";
     }
-    if (groups.length === 1) {
-        return `\nWHERE\n  ${groups[0]}`;
+    const sqlParts = where
+        .map(buildSqlFromGroup)
+        .filter((part) => part.length > 0);
+    if (sqlParts.length === 0) {
+        return "";
     }
-    return `\nWHERE\n  ${groups.join(" OR\n  ")}`;
+    return `\nWHERE\n  ${sqlParts.join("\n  ")}`;
 }
 function quoteIdentifier(identifier) {
     const safe = identifier.replace(/"/g, "");

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@narrative.io/data-collaboration-sdk-ts",
-	"version": "2.67.0",
+	"version": "2.69.0",
 	"main": "build/index.js",
 	"repository": "github:narrative-io/data-collaboration-sdk-ts",
 	"source": "src/index.ts",
@@ -33,7 +33,7 @@
 		"@types/jest": "30.0.0",
 		"babel-jest": "30.2.0",
 		"jest": "30.2.0",
-		"lefthook": "1.13.6",
+		"lefthook": "2.0.0",
 		"ts-jest": "29.4.4"
 	},
 	"dependencies": {