corebasic 1.0.205 → 1.0.208

package/libs/dip/suffix/date.js

@@ -0,0 +1,103 @@
+import {getNormalizedBounds} from './index.js'
+
+
+const MONTH_SHORT = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]
+const MONTH_LONG = ["January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December"]
+
+export function formatDate(value, format) {
+    const date = new Date(value)
+
+    if (Number.isNaN(date.getTime()))
+        throw new Error(`Error: Invalid date: ${value} in suffix policy specified for key in Dip`)
+
+
+    const day = date.getDate();
+    const monthIndex = date.getMonth();
+    const year = date.getFullYear();
+
+    // Pad single digits with a leading zero
+    const dd = String(day).padStart(2, '0');
+    const mm = String(monthIndex + 1).padStart(2, '0');
+    const yy = String(year).slice(-2);
+
+
+    // return format
+    // .replace('YYYY', year)
+    // .replace('YY', yy)
+    // .replace('MMMM', MONTH_LONG[monthIndex])
+    // .replace('MMM', MONTH_SHORT[monthIndex])
+    // .replace('MM', mm)
+    // .replace('DD', dd)
+    // .replace('D', day);
+
+
+    const replacements = {
+        YYYY: String(year),
+        YY: String(year).slice(-2),
+        MMMM: MONTH_LONG[monthIndex],
+        MMM: MONTH_SHORT[monthIndex],
+        MM: String(monthIndex + 1).padStart(2, "0"),
+        DD: String(day).padStart(2, "0"),
+        D: String(day),
+    };
+
+    return format.replace(/YYYY|MMMM|MMM|YY|MM|DD|D/g, token => replacements[token]);
+}
+
+
+// 📋 Supported Formats
+// YYYY: 4-digit year (e.g., 2026)
+// YY: 2-digit year (e.g., 26)
+// MMMM: Full month name (e.g., July)
+// MMM: Short month name (e.g., Jul)
+// MM: 2-digit padded month (e.g., 07)
+// DD: 2-digit padded day (e.g., 09)
+// D: Unpadded day (e.g., 9)
+
+
+
+// // let a = new Date()
+// // let a = "2026-11-1"
+// let a = new Date().getTime()
+// let b = formatDate(a, "YYYY/DD-MMMM")
+// console.log(b)
+//
+
+
+
+export function fillDates(bounds_t, format) {
+    const result = [];
+
+    let step;
+    if (format.includes("D"))
+        step = "day";
+    else if (format.includes("M"))
+        step = "month";
+    else
+        step = "year";
+
+    let bounds = { ...bounds_t, from: new Date(bounds_t.from), to: new Date(bounds_t.to) }
+
+    bounds = getNormalizedBounds(bounds, "date", step)
+
+    while (bounds.from <= bounds.to) {
+        result.push(formatDate(bounds.from, format));
+
+        switch (step) {
+            case "day":
+                bounds.from.setDate(bounds.from.getDate() + 1);
+                break;
+            case "month":
+                bounds.from.setMonth(bounds.from.getMonth() + 1);
+                break;
+            case "year":
+                bounds.from.setFullYear(bounds.from.getFullYear() + 1);
+                break;
+        }
+    }
+
+    return [...new Set(result)];
+}
+
+
+

package/libs/dip/suffix/index.js

@@ -0,0 +1,249 @@
+
+
+
+function flatten_id_values(target_key, values) {
+    let flattened = [];
+    flatten_value(target_key, values, flattened, false);
+    return flattened
+}
+
+function flatten_value(target_key, value, flattened, ancestor_has_id) {
+    const is_array = Array.isArray(value)
+    const is_object = value !== null && !is_array && typeof value === "object";
+    const is_regular = typeof value === "boolean" || typeof value === "number" || typeof value === "string"
+    const map = value
+    const arr = value
+
+    if (is_regular) {
+        flattened.push(value);
+    } else if (is_array) {
+        for (let v of arr) {
+            flatten_value(target_key, v, flattened, ancestor_has_id);
+        }
+    } else if (is_object) {
+
+        // {$nin}, {$not: {$in: []}} & $nor means Exclude. Indexes ensures everything else is excluded. $nin & $in with same id can be excluded but smells bad query thus punishable and so must incur the cost of fetching.
+
+        // Ensure _id is processed before entering the loop, otherwise a query { _id: 1, $or: [{_id: 2}, {}] } will skip _id: 1
+        if (!ancestor_has_id && Object.hasOwn(map, target_key)) {
+            flatten_value(target_key, map[target_key], flattened, true); // ancestor_has_id: true safely skips {_id: {_id: 1}} situation
+        }
+
+
+        for (let key in map) {
+            let val = map[key]
+
+            if (key === "$eq") {
+                flatten_value(target_key, val, flattened, ancestor_has_id);
+            } else if (key == "$gt" || key === "$gte" || key === "$lt" || key === "$lte") {
+                flattened.push({[key]: val});
+            } else if (key == "$in") {
+                // Only select $in, $eq, or (direct value which is done above in the match value section)
+                let arr = val
+                if (Array.isArray(arr)) {
+                    for (let v of arr) {
+                        flatten_value(target_key, v, flattened, ancestor_has_id);
+                    }
+                }
+            } else if (key === "$and" && Array.isArray(val)) {
+                flatten_value(target_key, val, flattened, ancestor_has_id); // recursively process all values
+            } else if (key === "$or" && Array.isArray(val)) {
+                // Store prior count.
+                let checkpoint = flattened.length;
+                // Every branch must produce 1 or more candidate _id.
+                // Recursively, inner $or rollback/capture correctly affects outer $or rollback/capture. So works correctly.
+                let arr = val
+                if (Array.isArray(arr)) {
+                    for (let branch of arr) {
+                        let before = flattened.length;
+
+                        flatten_value(target_key, branch, flattened, ancestor_has_id);
+
+                        if (flattened.length === before) {
+                            // branch produced no candidate _ids. Rollback to checkpoint.
+                            flattened.length = checkpoint;
+                            return;
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+
+export function entries(target_key, query, binary_slice) {
+    let result  = flatten_id_values(target_key, query); // extract all valid _id
+
+    // let cows = result.map(v => extract_direct_id(v, binary_slice))
+
+    // // cows.sort_by(|a, b| a.as_ref().cmp(b.as_ref())); // sort for improved forward processing because kv engine keys are already sorted.
+    // cows.sort_unstable_by(|a, b| a.as_ref().cmp(b.as_ref())); // sort for improved forward processing because kv engine keys are already sorted.
+    //
+    // // unique_bytes(cows)
+    // unique_sorted_bytes(cows) // Leverage already sorted so only need comparisons with the previous item.
+
+    const objects = result.filter(item => typeof item === "object")
+    const values = result.filter(item => typeof item !== "object")
+
+    let unique = [...new Set(values)];
+    return unique.concat(objects)
+}
+
+
+// NOTE:
+// reduceRanges() intentionally computes a conservative (wider) approximation.
+// It is used only for suffix policy planning, where false positives are acceptable
+// (extra suffix scans) but false negatives are not.
+export function reduceRanges(values) {
+    let gt = null;
+    let gte = null;
+    let lt = null;
+    let lte = null;
+    const other = [];
+
+    for (const value of values) {
+        if (typeof value !== "object" || value === null || Array.isArray(value)) {
+            other.push(value);
+            continue;
+        }
+
+        if ("$gt" in value) {
+            if (gt === null || value.$gt < gt)
+                gt = value.$gt;
+        } else if ("$gte" in value) {
+            if (gte === null || value.$gte < gte)
+                gte = value.$gte;
+        } else if ("$lt" in value) {
+            if (lt === null || value.$lt > lt)
+                lt = value.$lt;
+        } else if ("$lte" in value) {
+            if (lte === null || value.$lte > lte)
+                lte = value.$lte;
+        } else {
+            other.push(value);
+        }
+    }
+
+    // Resolve lower bound.
+    if (gt !== null || gte !== null) {
+        if (gt === null) {
+            other.push({ $gte: gte });
+        } else if (gte === null) {
+            other.push({ $gt: gt });
+        } else if (gt < gte) {
+            other.push({ $gt: gt });
+        } else if (gte < gt) {
+            other.push({ $gte: gte });
+        } else {
+            // Same value: inclusive is wider.
+            other.push({ $gte: gt });
+        }
+    }
+
+    // Resolve upper bound.
+    if (lt !== null || lte !== null) {
+        if (lt === null) {
+            other.push({ $lte: lte });
+        } else if (lte === null) {
+            other.push({ $lt: lt });
+        } else if (lt > lte) {
+            other.push({ $lt: lt });
+        } else if (lte > lt) {
+            other.push({ $lte: lte });
+        } else {
+            // Same value: inclusive is wider.
+            other.push({ $lte: lt });
+        }
+    }
+
+    return other;
+}
+
+// reduceRanges() resolves conflicting bounds before extractBounds() runs.
+// extractBounds() receives at most one lower bound and atmost one upper bound. reduceRanges() ensures this
+// The for loop in extractBounds() may look slightly misleading because it looks like it is designed to resolve conflicts: but reduceRanges() has already done the conflict resolution.
+// NOTE CRITICAL: extractBounds() MUST ONLY BE RUN ON THE OUTPUT OF reduceRanges()
+export function extractBounds(values) {
+    let lower = null;
+    let upper = null;
+    let lowerOp = null
+    let upperOp = null
+    const remaining = [];
+
+    for (const value of values) {
+        if (typeof value !== "object" || value === null || Array.isArray(value)) {
+            remaining.push(value);
+            continue;
+        }
+
+        if ("$gt" in value) {
+            lower = value.$gt;
+            lowerOp = "$gt"
+        } else if ("$gte" in value) {
+            lower = value.$gte;
+            lowerOp = "$gte"
+        } else if ("$lt" in value) {
+            upper = value.$lt;
+            upperOp = "$lt"
+        } else if ("$lte" in value) {
+            upper = value.$lte;
+            upperOp = "$lte"
+        } else {
+            remaining.push(value);
+        }
+    }
+
+    return {
+        values: remaining,
+        from: lower,
+        to: upper,
+        fromOp: lowerOp,
+        toOp: upperOp,
+    };
+}
+
+// Converts exclusive bounds ($gt/$lt) into inclusive bounds. Mutates if date type
+export function getNormalizedBounds(bounds_t, type, step) {
+    let bounds = {...bounds_t}
+
+    if (type === "number") {
+        if (bounds.fromOp === "$gt")
+            bounds.from++;
+        if (bounds.toOp === "$lt")
+            bounds.to--;
+    } else if (type === "date") {
+        if (step === "day") {
+            if (bounds.fromOp === "$gt")
+                bounds.from.setDate(bounds.from.getDate() + 1);
+            if (bounds.toOp === "$lt")
+                bounds.to.setDate(bounds.to.getDate() - 1);
+        } else if (step === "month") {
+            if (bounds.fromOp === "$gt")
+                bounds.from.setMonth(bounds.from.getMonth() + 1);
+            if (bounds.toOp === "$lt")
+                bounds.to.setMonth(bounds.to.getMonth() - 1);
+        } else {
+            if (bounds.fromOp === "$gt")
+                bounds.from.setFullYear(bounds.from.getFullYear() + 1);
+            if (bounds.toOp === "$lt")
+                bounds.to.setFullYear(bounds.to.getFullYear() - 1);
+        }
+    }
+
+    return bounds
+}
+
+// console.log('=============')
+// let a = entries("date", {
+//     $or: [
+//         {date: {$gt: 10, $lt: 20}},
+//         {date: {$gt: 3, $lt: 90}},
+//         {date: {$gt: 15, $lt: 25}},
+//         {date: 5},
+//         // {},
+//         {date: 5},
+//     ]
+// })
+// a = reduce_ranges(a)
+// console.log(a)

package/libs/dip/suffix/policy.js

@@ -0,0 +1,156 @@
+import {entries, reduceRanges, extractBounds} from './index.js'
+import {suffix} from './suffix.js'
+
+
+
+
+export function overlaps(a, b) {
+    const aFrom = a.from ?? -Infinity;
+    const aTo   = a.to   ??  Infinity;
+
+    const bFrom = b.from ?? -Infinity;
+    const bTo   = b.to   ??  Infinity;
+
+
+
+    const left = Math.max(aFrom, bFrom);
+    const right = Math.min(aTo, bTo);
+
+    if (left < right)
+        return true;
+
+    if (left > right)
+        return false;
+
+    // left === right, touching at one point.
+    const leftClosed =
+    (left === aFrom ? a.fromOp === "$gte" : a.toOp === "$lte") &&
+    (left === bFrom ? b.fromOp === "$gte" : b.toOp === "$lte");
+
+    return leftClosed;
+
+    // return Math.max(aFrom, bFrom) <= Math.min(aTo, bTo);
+}
+
+
+
+
+export function contains(bounds, value) {
+    const from = bounds.from ?? -Infinity;
+    const to   = bounds.to   ??  Infinity;
+
+    if (value < from)
+        return false;
+
+    if (value > to)
+        return false;
+
+    if (value === from && bounds.fromOp === "$gt")
+        return false;
+
+    if (value === to && bounds.toOp === "$lt")
+        return false;
+
+    return true;
+
+    // return value >= from && value <= to;
+}
+
+export function intersects(w_values, w_bounds, q_values, q_bounds) {
+    // value ↔ value
+    for (const value of w_values) {
+        if (q_values.includes(value))
+            return true;
+    }
+
+    // value(when) ↔ range(query)
+    if (q_bounds) {
+        for (const value of w_values) {
+            if (contains(q_bounds, value))
+                return true;
+        }
+    }
+
+    // value(query) ↔ range(when)
+    if (w_bounds) {
+        for (const value of q_values) {
+            if (contains(w_bounds, value))
+                return true;
+        }
+    }
+
+    // range ↔ range
+    if (w_bounds && q_bounds && overlaps(w_bounds, q_bounds))
+        return true;
+
+    return false;
+}
+
+
+// NOTE: Matching is based on intersection, not equality. A policy matches if, for every key in `when`, there exists at least one value that satisfies both the `when` constraint and the query constraint.
+export function matchPolicy(query, when, g_obj) {
+    let obj = g_obj ?? {}
+    for (let key in when) {
+        if (key === "$and" || key === "$or") {
+            for (const element of when[key]) {
+                matchPolicy(query, element, obj)
+            }
+        } else if (key === "$ne" || key === "$nin" || key === "$not" || (when[key] && typeof when[key] === "object" && ("$ne" in when[key] || "$nin" in when[key] || "$not" in when[key])) ) {
+            throw new Error(`Error: Policy 'when' does not support negative constraints $ne, $nin and $not in Dip`)
+            // NOTE:
+            // Negative constraints are invalid for policy selection. Cost is additional suffix scans. But we disallow it so that it is immediately communicated
+            // There's no suffix you can eliminate from just knowing a negative. The safest thing is exactly to not let it influence policy selection.
+        } else {
+            obj[key] = obj[key] ?? []
+            obj[key].push(...entries(key, when))
+        }
+    }
+
+    // Only compare once, on the outermost call.
+    if (g_obj)
+        return;
+
+    for (let key in obj) {
+        obj[key] = [...new Set(obj[key])];
+
+        let w_values = obj[key]
+
+        let w_ranges = w_values.filter(item =>  typeof item === "object")
+        w_values = w_values.filter(item =>  typeof item !== "object")
+        const w_bounds = w_ranges.length ? extractBounds(reduceRanges(w_ranges)) : null
+
+        let q_values = entries(key, query)
+        let q_ranges = q_values.filter(item =>  typeof item === "object")
+        q_values = q_values.filter(item =>  typeof item !== "object")
+        const q_bounds = q_ranges.length ? extractBounds(reduceRanges(q_ranges)) : null
+
+        if (!intersects(w_values, w_bounds, q_values, q_bounds))
+            return false;
+
+    }
+    return true
+}
+
+
+
+
+
+
+export async function applySuffixPolicy(COLLECTIONS_JSON, collection, query) {
+    const collection_policy = COLLECTIONS_JSON[collection]?.policy ?? []
+    const suffixPolicies = typeof collection_policy === "object" && !Array.isArray(collection_policy) ? [collection_policy] : COLLECTIONS_JSON[collection]?.policy ?? []
+    let suffixes = []
+    for (const policy of suffixPolicies) {
+        if (matchPolicy(query, policy.when ?? {})) {
+            // generate suffixes
+            const array = await suffix(query, policy.suffix ?? [])
+            suffixes.push(...array)
+        }
+    }
+    suffixes = [...new Set(suffixes)];
+    return suffixes
+}
+
+
+
+