corebasic 1.0.212 → 1.0.214

package/dist/libs/dip/suffix/index.js

@@ -0,0 +1,227 @@
+function flatten_id_values(target_key, values) {
+    let flattened = [];
+    flatten_value(target_key, values, flattened, false);
+    return flattened;
+}
+function flatten_value(target_key, value, flattened, ancestor_has_id) {
+    const is_array = Array.isArray(value);
+    const is_object = value !== null && !is_array && typeof value === "object";
+    const is_regular = typeof value === "boolean" || typeof value === "number" || typeof value === "string";
+    const map = value;
+    const arr = value;
+    if (is_regular) {
+        flattened.push(value);
+    }
+    else if (is_array) {
+        for (let v of arr) {
+            flatten_value(target_key, v, flattened, ancestor_has_id);
+        }
+    }
+    else if (is_object) {
+        // {$nin}, {$not: {$in: []}} & $nor means Exclude. Indexes ensures everything else is excluded. $nin & $in with same id can be excluded but smells bad query thus punishable and so must incur the cost of fetching.
+        // Ensure _id is processed before entering the loop, otherwise a query { _id: 1, $or: [{_id: 2}, {}] } will skip _id: 1
+        if (!ancestor_has_id && Object.hasOwn(map, target_key)) {
+            flatten_value(target_key, map[target_key], flattened, true); // ancestor_has_id: true safely skips {_id: {_id: 1}} situation
+        }
+        for (let key in map) {
+            let val = map[key];
+            if (key === "$eq") {
+                flatten_value(target_key, val, flattened, ancestor_has_id);
+            }
+            else if (key == "$gt" || key === "$gte" || key === "$lt" || key === "$lte") {
+                flattened.push({ [key]: val });
+            }
+            else if (key == "$in") {
+                // Only select $in, $eq, or (direct value which is done above in the match value section)
+                let arr = val;
+                if (Array.isArray(arr)) {
+                    for (let v of arr) {
+                        flatten_value(target_key, v, flattened, ancestor_has_id);
+                    }
+                }
+            }
+            else if (key === "$and" && Array.isArray(val)) {
+                flatten_value(target_key, val, flattened, ancestor_has_id); // recursively process all values
+            }
+            else if (key === "$or" && Array.isArray(val)) {
+                // Store prior count.
+                let checkpoint = flattened.length;
+                // Every branch must produce 1 or more candidate _id.
+                // Recursively, inner $or rollback/capture correctly affects outer $or rollback/capture. So works correctly.
+                let arr = val;
+                if (Array.isArray(arr)) {
+                    for (let branch of arr) {
+                        let before = flattened.length;
+                        flatten_value(target_key, branch, flattened, ancestor_has_id);
+                        if (flattened.length === before) {
+                            // branch produced no candidate _ids. Rollback to checkpoint.
+                            flattened.length = checkpoint;
+                            return;
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+export function entries(target_key, query, binary_slice) {
+    let result = flatten_id_values(target_key, query); // extract all valid _id
+    // let cows = result.map(v => extract_direct_id(v, binary_slice))
+    // // cows.sort_by(|a, b| a.as_ref().cmp(b.as_ref())); // sort for improved forward processing because kv engine keys are already sorted.
+    // cows.sort_unstable_by(|a, b| a.as_ref().cmp(b.as_ref())); // sort for improved forward processing because kv engine keys are already sorted.
+    //
+    // // unique_bytes(cows)
+    // unique_sorted_bytes(cows) // Leverage already sorted so only need comparisons with the previous item.
+    const objects = result.filter(item => typeof item === "object");
+    const values = result.filter(item => typeof item !== "object");
+    let unique = [...new Set(values)];
+    return unique.concat(objects);
+}
+// NOTE:
+// reduceRanges() intentionally computes a conservative (wider) approximation.
+// It is used only for suffix policy planning, where false positives are acceptable
+// (extra suffix scans) but false negatives are not.
+export function reduceRanges(values) {
+    let gt = null;
+    let gte = null;
+    let lt = null;
+    let lte = null;
+    const other = [];
+    for (const value of values) {
+        if (typeof value !== "object" || value === null || Array.isArray(value)) {
+            other.push(value);
+            continue;
+        }
+        if ("$gt" in value) {
+            if (gt === null || value.$gt < gt)
+                gt = value.$gt;
+        }
+        else if ("$gte" in value) {
+            if (gte === null || value.$gte < gte)
+                gte = value.$gte;
+        }
+        else if ("$lt" in value) {
+            if (lt === null || value.$lt > lt)
+                lt = value.$lt;
+        }
+        else if ("$lte" in value) {
+            if (lte === null || value.$lte > lte)
+                lte = value.$lte;
+        }
+        else {
+            other.push(value);
+        }
+    }
+    // Resolve lower bound.
+    if (gt !== null || gte !== null) {
+        if (gt === null) {
+            other.push({ $gte: gte });
+        }
+        else if (gte === null) {
+            other.push({ $gt: gt });
+        }
+        else if (gt < gte) {
+            other.push({ $gt: gt });
+        }
+        else if (gte < gt) {
+            other.push({ $gte: gte });
+        }
+        else {
+            // Same value: inclusive is wider.
+            other.push({ $gte: gt });
+        }
+    }
+    // Resolve upper bound.
+    if (lt !== null || lte !== null) {
+        if (lt === null) {
+            other.push({ $lte: lte });
+        }
+        else if (lte === null) {
+            other.push({ $lt: lt });
+        }
+        else if (lt > lte) {
+            other.push({ $lt: lt });
+        }
+        else if (lte > lt) {
+            other.push({ $lte: lte });
+        }
+        else {
+            // Same value: inclusive is wider.
+            other.push({ $lte: lt });
+        }
+    }
+    return other;
+}
+// reduceRanges() resolves conflicting bounds before extractBounds() runs.
+// extractBounds() receives at most one lower bound and atmost one upper bound. reduceRanges() ensures this
+// The for loop in extractBounds() may look slightly misleading because it looks like it is designed to resolve conflicts: but reduceRanges() has already done the conflict resolution.
+// NOTE CRITICAL: extractBounds() MUST ONLY BE RUN ON THE OUTPUT OF reduceRanges()
+export function extractBounds(values) {
+    let lower = null;
+    let upper = null;
+    let lowerOp = null;
+    let upperOp = null;
+    const remaining = [];
+    for (const value of values) {
+        if (typeof value !== "object" || value === null || Array.isArray(value)) {
+            remaining.push(value);
+            continue;
+        }
+        if ("$gt" in value) {
+            lower = value.$gt;
+            lowerOp = "$gt";
+        }
+        else if ("$gte" in value) {
+            lower = value.$gte;
+            lowerOp = "$gte";
+        }
+        else if ("$lt" in value) {
+            upper = value.$lt;
+            upperOp = "$lt";
+        }
+        else if ("$lte" in value) {
+            upper = value.$lte;
+            upperOp = "$lte";
+        }
+        else {
+            remaining.push(value);
+        }
+    }
+    return {
+        values: remaining,
+        from: lower,
+        to: upper,
+        fromOp: lowerOp,
+        toOp: upperOp,
+    };
+}
+// Converts exclusive bounds ($gt/$lt) into inclusive bounds
+export function getNormalizedBounds(bounds_t, type) {
+    let bounds = { ...bounds_t };
+    if (type === "number") {
+        if (bounds.fromOp === "$gt")
+            bounds.from++;
+        if (bounds.toOp === "$lt")
+            bounds.to--;
+    }
+    else if (type === "date") {
+        if (bounds.fromOp === "$gt")
+            bounds.from = new Date(bounds.from.getTime() + 1);
+        if (bounds.toOp === "$lt")
+            bounds.to = new Date(bounds.to.getTime() - 1);
+    }
+    return bounds;
+}
+// console.log('=============')
+// let a = entries("date", {
+//     $or: [
+//         {date: {$gt: 10, $lt: 20}},
+//         {date: {$gt: 3, $lt: 90}},
+//         {date: {$gt: 15, $lt: 25}},
+//         {date: 5},
+//         // {},
+//         {date: 5},
+//     ]
+// })
+// a = reduce_ranges(a)
+// console.log(a)

package/dist/libs/dip/suffix/policy.js

@@ -0,0 +1,113 @@
+import { entries, reduceRanges, extractBounds } from './index.js';
+import { suffix } from './suffix.js';
+export function overlaps(a, b) {
+    const aFrom = a.from ?? -Infinity;
+    const aTo = a.to ?? Infinity;
+    const bFrom = b.from ?? -Infinity;
+    const bTo = b.to ?? Infinity;
+    const left = Math.max(aFrom, bFrom);
+    const right = Math.min(aTo, bTo);
+    if (left < right)
+        return true;
+    if (left > right)
+        return false;
+    // left === right, touching at one point.
+    const leftClosed = (left === aFrom ? a.fromOp === "$gte" : a.toOp === "$lte") &&
+        (left === bFrom ? b.fromOp === "$gte" : b.toOp === "$lte");
+    return leftClosed;
+    // return Math.max(aFrom, bFrom) <= Math.min(aTo, bTo);
+}
+export function contains(bounds, value) {
+    const from = bounds.from ?? -Infinity;
+    const to = bounds.to ?? Infinity;
+    if (value < from)
+        return false;
+    if (value > to)
+        return false;
+    if (value === from && bounds.fromOp === "$gt")
+        return false;
+    if (value === to && bounds.toOp === "$lt")
+        return false;
+    return true;
+    // return value >= from && value <= to;
+}
+export function intersects(w_values, w_bounds, q_values, q_bounds) {
+    // value ↔ value
+    for (const value of w_values) {
+        if (q_values.includes(value))
+            return true;
+    }
+    // value(when) ↔ range(query)
+    if (q_bounds) {
+        for (const value of w_values) {
+            if (contains(q_bounds, value))
+                return true;
+        }
+    }
+    // value(query) ↔ range(when)
+    if (w_bounds) {
+        for (const value of q_values) {
+            if (contains(w_bounds, value))
+                return true;
+        }
+    }
+    // range ↔ range
+    if (w_bounds && q_bounds && overlaps(w_bounds, q_bounds))
+        return true;
+    return false;
+}
+// NOTE: Matching is based on intersection, not equality. A policy matches if, for every key in `when`, there exists at least one value that satisfies both the `when` constraint and the query constraint.
+export function matchPolicy(query, when, g_obj) {
+    let obj = g_obj ?? {};
+    for (let key in when) {
+        if (key === "$and" || key === "$or") {
+            for (const element of when[key]) {
+                matchPolicy(query, element, obj);
+            }
+        }
+        else if (key === "$ne" || key === "$nin" || key === "$not" || (when[key] && typeof when[key] === "object" && ("$ne" in when[key] || "$nin" in when[key] || "$not" in when[key]))) {
+            throw new Error(`Error: Policy 'when' does not support negative constraints $ne, $nin and $not in Dip`);
+            // NOTE:
+            // Negative constraints are invalid for policy selection. Cost is additional suffix scans. But we disallow it so that it is immediately communicated
+            // There's no suffix you can eliminate from just knowing a negative. The safest thing is exactly to not let it influence policy selection.
+        }
+        else {
+            obj[key] = obj[key] ?? [];
+            obj[key].push(...entries(key, when));
+        }
+    }
+    // Only compare once, on the outermost call.
+    if (g_obj)
+        return;
+    for (let key in obj) {
+        obj[key] = [...new Set(obj[key])];
+        let w_values = obj[key];
+        let w_ranges = w_values.filter(item => typeof item === "object");
+        w_values = w_values.filter(item => typeof item !== "object");
+        const w_bounds = w_ranges.length ? extractBounds(reduceRanges(w_ranges)) : null;
+        let q_values = entries(key, query);
+        let q_ranges = q_values.filter(item => typeof item === "object");
+        q_values = q_values.filter(item => typeof item !== "object");
+        const q_bounds = q_ranges.length ? extractBounds(reduceRanges(q_ranges)) : null;
+        if (!intersects(w_values, w_bounds, q_values, q_bounds))
+            return false;
+    }
+    return true;
+}
+export async function applySuffixPolicy(COLLECTIONS_JSON, collections, query, insertMode, arg) {
+    collections = Array.isArray(collections) ? collections : [collections];
+    let suffixes = [];
+    for (let collection of collections) {
+        const collection_policy = COLLECTIONS_JSON[collection]?.policy ?? [];
+        const suffixPolicies = typeof collection_policy === "object" && !Array.isArray(collection_policy) ? [collection_policy] : COLLECTIONS_JSON[collection]?.policy ?? [];
+        for (const policy of suffixPolicies) {
+            if (matchPolicy(query, policy.when ?? {})) {
+                // generate suffixes
+                const array = await suffix(query, policy.suffix ?? [], insertMode, arg);
+                suffixes.push(...array);
+            }
+        }
+        suffixes = [...new Set(suffixes)];
+    }
+    return suffixes;
+}

package/dist/libs/dip/suffix/query.js

@@ -0,0 +1,279 @@
+// Lightweight Query Matcher (JavaScript)
+// ========= Utility Functions =========
+function toF64(value) {
+    if (value === null || value === undefined)
+        return 0;
+    if (typeof value === 'number')
+        return value;
+    if (typeof value === 'string') {
+        const parsed = parseFloat(value);
+        return isNaN(parsed) ? 0 : parsed;
+    }
+    if (typeof value === 'boolean')
+        return value ? 1 : 0;
+    return 0;
+}
+function getValueAt(obj, key) {
+    if (obj === null || obj === undefined)
+        return null;
+    if (typeof obj !== 'object')
+        return null;
+    return obj[key] !== undefined ? obj[key] : null;
+}
+// ========= Comparison Primitives =========
+function eqQuery(v1, v2) {
+    // Deep equality for objects/arrays, strict for primitives
+    if (v1 === v2)
+        return true;
+    if (v1 === null || v2 === null)
+        return v1 === v2;
+    if (typeof v1 !== typeof v2) {
+        // Try numeric comparison
+        if (typeof v1 === 'number' || typeof v2 === 'number') {
+            return toF64(v1) === toF64(v2);
+        }
+        return false;
+    }
+    if (typeof v1 === 'object') {
+        if (Array.isArray(v1) !== Array.isArray(v2))
+            return false;
+        if (Array.isArray(v1)) {
+            if (v1.length !== v2.length)
+                return false;
+            for (let i = 0; i < v1.length; i++) {
+                if (!eqQuery(v1[i], v2[i]))
+                    return false;
+            }
+            return true;
+        }
+        const keys1 = Object.keys(v1);
+        const keys2 = Object.keys(v2);
+        if (keys1.length !== keys2.length)
+            return false;
+        for (const key of keys1) {
+            if (!keys2.includes(key))
+                return false;
+            if (!eqQuery(v1[key], v2[key]))
+                return false;
+        }
+        return true;
+    }
+    return false;
+}
+function gtQuery(v1, v2) {
+    return toF64(v1) > toF64(v2);
+}
+function gteQuery(v1, v2) {
+    return toF64(v1) >= toF64(v2);
+}
+function ltQuery(v1, v2) {
+    return toF64(v1) < toF64(v2);
+}
+function lteQuery(v1, v2) {
+    return toF64(v1) <= toF64(v2);
+}
+function existsQuery(v1, v2) {
+    const exists = v1 !== null && v1 !== undefined;
+    const condition = typeof v2 === 'boolean' ? v2 : true;
+    return condition === exists;
+}
+function beginQuery(input, arg) {
+    if (typeof input === 'string' && typeof arg === 'string') {
+        return input.startsWith(arg);
+    }
+    return false;
+}
+function endQuery(input, arg) {
+    if (typeof input === 'string' && typeof arg === 'string') {
+        return input.endsWith(arg);
+    }
+    return false;
+}
+function containsQuery(input, arg, caseSensitive = true) {
+    if (typeof input !== 'string')
+        return false;
+    if (Array.isArray(arg)) {
+        if (arg.length === 0)
+            return false;
+        let allMatch = true;
+        for (const v of arg) {
+            let needle = typeof v === 'string' ? v : String(v);
+            let haystack = input;
+            if (!caseSensitive) {
+                needle = needle.toLowerCase();
+                haystack = haystack.toLowerCase();
+            }
+            if (!haystack.includes(needle)) {
+                allMatch = false;
+                break;
+            }
+        }
+        return allMatch;
+    }
+    if (typeof arg === 'string') {
+        let needle = arg;
+        let haystack = input;
+        if (!caseSensitive) {
+            needle = needle.toLowerCase();
+            haystack = haystack.toLowerCase();
+        }
+        return haystack.includes(needle);
+    }
+    return false;
+}
+// ========= Query Commands =========
+function inQuery(input, args) {
+    if (Array.isArray(args)) {
+        for (const item of args) {
+            if (eqQuery(input, item))
+                return true;
+        }
+        return false;
+    }
+    if (args !== null && typeof args === 'object')
+        return false;
+    return eqQuery(input, args);
+}
+function elemMatchQuery(input, query) {
+    if (!Array.isArray(input))
+        return false;
+    const term = query.$ || '$';
+    const captureLimit = query.$captureLimit || 0;
+    let countMatched = 0;
+    for (let index = 0; index < input.length; index++) {
+        if (andQuery(input[index], query)) {
+            if (captureLimit > 0 && countMatched >= captureLimit)
+                break;
+            countMatched++;
+        }
+    }
+    return countMatched > 0;
+}
+function lengthQuery(input, arg) {
+    let len;
+    if (Array.isArray(input)) {
+        len = input.length;
+    }
+    else if (typeof input === 'string') {
+        len = input.length;
+    }
+    else {
+        return false;
+    }
+    if (arg !== null && typeof arg === 'object' && !Array.isArray(arg)) {
+        return andQuery(len, arg);
+    }
+    if (typeof arg === 'number') {
+        return len === arg;
+    }
+    return false;
+}
+function icaseQuery(input, arg) {
+    if (typeof input !== 'string')
+        return false;
+    const lowerInput = input.toLowerCase();
+    if (arg !== null && typeof arg === 'object' && !Array.isArray(arg)) {
+        return andQuery(lowerInput, arg);
+    }
+    if (typeof arg === 'string') {
+        return eqQuery(lowerInput, arg.toLowerCase());
+    }
+    return false;
+}
+function selectCommand(key, input, value) {
+    switch (key) {
+        // Comparison
+        case '$':
+            return true;
+        case '$eq':
+            return eqQuery(input, value);
+        case '$ne':
+            return !eqQuery(input, value);
+        case '$gt':
+            return gtQuery(input, value);
+        case '$gte':
+            return gteQuery(input, value);
+        case '$lt':
+            return ltQuery(input, value);
+        case '$lte':
+            return lteQuery(input, value);
+        case '$in':
+            return inQuery(input, value);
+        case '$nin':
+            return !inQuery(input, value);
+        // Logical
+        case '$not':
+            return !andQuery(input, value);
+        case '$exists':
+            return existsQuery(input, value);
+        case '$or':
+            return orQuery(input, value);
+        case '$nor':
+            return !orQuery(input, value);
+        case '$and':
+            if (!Array.isArray(value))
+                return false;
+            for (const item of value) {
+                if (!andQuery(input, item))
+                    return false;
+            }
+            return true;
+        // Array
+        case '$elemMatch':
+            return elemMatchQuery(input, value);
+        case '$length':
+            return lengthQuery(input, value);
+        // String matching
+        case '$icase':
+            return icaseQuery(input, value);
+        case '$begin':
+            return beginQuery(input, value);
+        case '$end':
+            return endQuery(input, value);
+        case '$contains':
+            return containsQuery(input, value);
+        default:
+            if (key.startsWith('$'))
+                return true; // Unknown $keys pass through
+            return false;
+    }
+}
+function orQuery(input, value) {
+    if (!Array.isArray(value))
+        return false;
+    for (const item of value) {
+        if (andQuery(input, item))
+            return true;
+    }
+    return false;
+}
+function andQuery(input, qvalue) {
+    if (qvalue === null || qvalue === undefined) {
+        return eqQuery(input, qvalue);
+    }
+    if (typeof qvalue === 'object' && !Array.isArray(qvalue)) {
+        for (const [key, value] of Object.entries(qvalue)) {
+            if (key.startsWith('$')) {
+                if (!selectCommand(key, input, value)) {
+                    return false;
+                }
+            }
+            else {
+                const v = (typeof value === 'object' && value !== null && !Array.isArray(value))
+                    ? andQuery(getValueAt(input, key), value)
+                    : eqQuery(getValueAt(input, key), value);
+                if (!v)
+                    return false;
+            }
+        }
+        return true;
+    }
+    if (Array.isArray(qvalue)) {
+        return false;
+    }
+    return eqQuery(input, qvalue);
+}
+// ========= Main Entry Point =========
+export function search(data, query) {
+    return andQuery(data, query);
+}