@graffy/pg 0.19.0 → 0.19.1-alpha.1

package/{types/Db.d.ts → Db.d.ts}

@@ -1,6 +1,5 @@
 export default class Db {
     constructor(connection: any);
-    client: any;
     query(sql: any, tableOptions: any): Promise<any>;
     readSql(sql: any, tableOptions: any): Promise<any>;
     writeSql(sql: any, tableOptions: any): Promise<any>;

package/Db.js

@@ -0,0 +1,236 @@
+import { cmp, decodeArgs, decodeGraph, decodeQuery, encodeGraph, encodePath, finalize, isEmpty, isPlainObject, isRange, merge, mergeObject, unwrap, wrap, wrapObject, } from '@graffy/common';
+import debug from 'debug';
+import pg from 'pg';
+import sqlTag from 'sql-template-tag';
+import formatSql from "./sql/format.js";
+import { del, patch, put, selectByArgs, selectByIds } from "./sql/index.js";
+const log = debug('graffy:pg:db');
+const { Pool, Client, types } = pg;
+export default class Db {
+    constructor(connection) {
+        if (typeof connection === 'object' &&
+            connection &&
+            (connection instanceof Pool || connection instanceof Client)) {
+            this.client = connection;
+        }
+        else {
+            this.client = new Pool(connection);
+        }
+    }
+    async query(sql, tableOptions) {
+        log(`Making SQL query: ${sql.text}`, sql.values);
+        const cubeOid = Number.parseInt(tableOptions?.schema?.typeOids?.cube || '0', 10) || null;
+        try {
+            sql.types = {
+                getTypeParser: (oid, format) => {
+                    if (oid === types.builtins.INT8) {
+                        return (value) => Number.parseInt(value, 10);
+                    }
+                    if (oid === cubeOid) {
+                        return (value) => {
+                            const array = value
+                                .slice(1, -1)
+                                .split(/\)\s*,\s*\(/)
+                                .map((corner) => corner
+                                .split(',')
+                                .map((coord) => Number.parseFloat(coord.trim())));
+                            return array.length > 1 ? array : array[0];
+                        };
+                    }
+                    return types.getTypeParser(oid, format);
+                },
+            };
+            return await this.client.query(sql);
+        }
+        catch (e) {
+            const message = [
+                e.message,
+                e.detail,
+                e.hint,
+                e.where,
+                sql.text,
+                JSON.stringify(sql.values),
+            ]
+                .filter(Boolean)
+                .join('; ');
+            throw Error(`pg.sql_error ${message}`);
+        }
+    }
+    async readSql(sql, tableOptions) {
+        const result = (await this.query(sql, tableOptions)).rows;
+        // Each row is an array, as there is only one column returned.
+        log('Read result', result);
+        return result;
+    }
+    async writeSql(sql, tableOptions) {
+        const res = await this.query(sql, tableOptions);
+        log('Rows written', res.rowCount);
+        if (!res.rowCount) {
+            throw Error(`pg.nothing_written ${sql.text} with ${sql.values}`);
+        }
+        return res.rows;
+    }
+    /*
+      Adds .schema to tableOptions if it doesn't exist yet.
+      It mutates the argument, to "persist" the results and
+      avoid this query in every operation.
+    */
+    async ensureSchema(tableOptions, typeOids) {
+        if (tableOptions.schema)
+            return;
+        const { table, verCol, joins } = tableOptions;
+        const tableInfoSchema = (await this.query(sqlTag `
+        SELECT table_schema, table_type
+        FROM information_schema.tables
+        WHERE table_name = ${table}
+        ORDER BY array_position(current_schemas(false)::text[], table_schema::text) ASC
+        LIMIT 1`)).rows[0];
+        const tableSchema = tableInfoSchema.table_schema;
+        const tableType = tableInfoSchema.table_type;
+        const types = (await this.query(sqlTag `
+        SELECT jsonb_object_agg(column_name, udt_name) AS column_types
+        FROM information_schema.columns
+        WHERE
+          table_name = ${table} AND
+          table_schema = ${tableSchema}`)).rows[0].column_types;
+        if (!types)
+            throw Error(`pg.missing_table ${table}`);
+        typeOids =
+            typeOids ||
+                (await this.query(sqlTag `
+        SELECT jsonb_object_agg(typname, oid) AS type_oids
+        FROM pg_type
+        WHERE typname = 'cube'`)).rows[0].type_oids;
+        // console.log({ typeOids });
+        const verDefault = (await this.query(sqlTag `
+        SELECT column_default
+        FROM information_schema.columns
+        WHERE
+          table_name = ${table} AND
+          table_schema = ${tableSchema} AND
+          column_name = ${verCol}`)).rows[0].column_default;
+        if (!verDefault && tableType !== 'VIEW') {
+            throw Error(`pg.verCol_without_default ${verCol}`);
+        }
+        await Promise.all(Object.values(joins).map((joinOptions) => this.ensureSchema(joinOptions, typeOids)));
+        log('ensureSchema', types);
+        tableOptions.schema = { types, typeOids };
+        tableOptions.verDefault = verDefault;
+    }
+    async read(rootQuery, tableOptions) {
+        const idQueries = {};
+        const promises = [];
+        const results = [];
+        const { prefix: rawPrefix } = tableOptions;
+        const prefix = encodePath(rawPrefix);
+        await this.ensureSchema(tableOptions);
+        const getByArgs = async (args, projection) => {
+            const sql = selectByArgs(args, projection, tableOptions);
+            const result = await this.readSql(sql, tableOptions);
+            const wrappedGraph = encodeGraph(wrapObject(result, rawPrefix));
+            log('getByArgs', wrappedGraph);
+            merge(results, wrappedGraph);
+        };
+        const explainArgs = async (args, _projection) => {
+            const { analyze, $explain: qArgs } = args;
+            const qSql = selectByArgs(qArgs, null, tableOptions);
+            const sql = sqlTag `EXPLAIN (${analyze ? sqlTag `ANALYZE, BUFFERS, TIMING, ` : sqlTag ``}COSTS, VERBOSE, FORMAT JSON) ${qSql}`;
+            const result = await this.readSql(sql, tableOptions);
+            const wrappedGraph = encodeGraph(wrapObject({
+                $key: args,
+                sql: formatSql(qSql),
+                plan: result[0]['QUERY PLAN'][0],
+            }, rawPrefix));
+            log('explainArgs', wrappedGraph);
+            merge(results, wrappedGraph);
+        };
+        const getByIds = async () => {
+            // TODO: Calculate a combined projection.
+            // Bonus: Strategically split into multiple read operations
+            // based on projection.
+            const sql = selectByIds(Object.keys(idQueries), null, tableOptions);
+            const result = await this.readSql(sql, tableOptions);
+            for (const object of result) {
+                const wrappedGraph = encodeGraph(wrapObject(object, rawPrefix));
+                log('getByIds', wrappedGraph);
+                merge(results, wrappedGraph);
+            }
+        };
+        const query = unwrap(rootQuery, prefix);
+        for (const node of query) {
+            const args = decodeArgs(node);
+            if (isPlainObject(args)) {
+                if (node.prefix) {
+                    for (const childNode of node.children) {
+                        const childArgs = decodeArgs(childNode);
+                        const projection = childNode.children
+                            ? decodeQuery(childNode.children)
+                            : null;
+                        promises.push(getByArgs({ ...args, ...childArgs }, projection));
+                    }
+                }
+                else {
+                    const projection = node.children ? decodeQuery(node.children) : null;
+                    if (args.$explain) {
+                        promises.push(explainArgs(args, projection));
+                    }
+                    else {
+                        promises.push(getByArgs(args, projection));
+                    }
+                }
+            }
+            else {
+                idQueries[args] = node.children;
+            }
+        }
+        if (!isEmpty(idQueries))
+            promises.push(getByIds());
+        await Promise.all(promises);
+        log('dbRead', rootQuery, results);
+        return finalize(results, wrap(query, prefix));
+    }
+    async write(rootChange, tableOptions) {
+        const { prefix: rawPrefix } = tableOptions;
+        const prefix = encodePath(rawPrefix);
+        await this.ensureSchema(tableOptions);
+        const change = unwrap(rootChange, prefix);
+        const puts = [];
+        const sqls = [];
+        for (const node of change) {
+            const arg = decodeArgs(node);
+            if (isRange(node)) {
+                if (cmp(node.key, node.end) === 0) {
+                    log('delete', node);
+                    sqls.push(del(arg, tableOptions));
+                    continue;
+                }
+                throw Error('pg_write.write_range_unsupported');
+            }
+            const object = decodeGraph(node.children);
+            if (isPlainObject(arg)) {
+                mergeObject(object, arg);
+            }
+            else {
+                object[tableOptions.idCol] = arg;
+            }
+            if (object.$put && object.$put !== true) {
+                throw Error('pg_write.partial_put_unsupported');
+            }
+            if (object.$put) {
+                puts.push([object, arg]);
+            }
+            else {
+                sqls.push(patch(object, arg, tableOptions));
+            }
+        }
+        if (puts.length)
+            sqls.push(...put(puts, tableOptions));
+        const result = [];
+        await Promise.all(sqls.map((sql) => this.writeSql(sql, tableOptions).then((object) => {
+            log('returned_object_wrapped', wrapObject(object, rawPrefix));
+            merge(result, encodeGraph(wrapObject(object, rawPrefix)));
+        })));
+        log('dbWrite', rootChange, result);
+        return result;
+    }
+}

package/filter/filterObject.js

@@ -0,0 +1,39 @@
+import { encodePath, unwrapObject } from '@graffy/common';
+import getAst from "./getAst.js";
+export default function filterObject(filter, object) {
+    function lookup(path) {
+        return unwrapObject(object, encodePath(path));
+    }
+    function checkNode(ast) {
+        switch (ast[0]) {
+            case '$eq':
+                return lookup(ast[1]) === ast[2];
+            case '$ne':
+                return lookup(ast[1]) !== ast[2];
+            case '$lt':
+                return lookup(ast[1]) < ast[2];
+            case '$lte':
+                return lookup(ast[1]) <= ast[2];
+            case '$gt':
+                return lookup(ast[1]) > ast[2];
+            case '$gte':
+                return lookup(ast[1]) >= ast[2];
+            case '$in':
+                return Array.isArray(ast[2]) && ast[2].includes(lookup(ast[1]));
+            case '$nin':
+                return !(Array.isArray(ast[2]) && ast[2].includes(lookup(ast[1])));
+            case '$cts':
+            case '$ctd':
+            case '$ovp':
+            case '$and':
+            case '$or':
+            case '$not':
+            case '$any':
+            case '$all':
+            case '$has':
+                throw Error('pgfilter.unimplemented');
+        }
+    }
+    const rootAst = getAst(filter);
+    return checkNode(rootAst);
+}

package/filter/getAst.js

@@ -0,0 +1,152 @@
+const valid = {
+    $eq: true,
+    $lt: true,
+    $gt: true,
+    $lte: true,
+    $gte: true,
+    $re: true,
+    $ire: true,
+    $text: true,
+    $and: true,
+    $or: true,
+    $any: true,
+    $all: true,
+    $has: true,
+    $cts: true,
+    $ctd: true,
+    $keycts: true,
+    $keyctd: true,
+};
+const inverse = {
+    $eq: '$neq',
+    $neq: '$eq',
+    $in: '$nin',
+    $nin: '$in',
+    $lt: '$gte',
+    $gte: '$lt',
+    $gt: '$lte',
+    $lte: '$gt',
+};
+export default function getAst(filter) {
+    return simplify(construct(filter));
+}
+function isValidSubQuery(node) {
+    if (!node || typeof node !== 'object')
+        return false;
+    const keys = Object.keys(node);
+    for (const key of keys) {
+        if (key[0] === '$' && !['$and', '$or', '$not'].includes(key))
+            return false;
+        if (key[0] !== '$')
+            return true;
+    }
+    for (const key in node) {
+        if (!isValidSubQuery(node[key]))
+            return false;
+    }
+    return false;
+}
+function construct(node, prop, op) {
+    if (!node || typeof node !== 'object' || (prop && op)) {
+        if (op && prop)
+            return [op, prop, node];
+        if (prop)
+            return ['$eq', prop, node];
+        throw Error(`pgast.expected_prop_before:${JSON.stringify(node)}`);
+    }
+    if (Array.isArray(node)) {
+        return ['$or', node.map((item) => construct(item, prop, op))];
+    }
+    if (prop && isValidSubQuery(node)) {
+        return ['$sub', prop, construct(node)];
+    }
+    return [
+        '$and',
+        Object.entries(node).map(([key, val]) => {
+            if (key === '$or' || key === '$and') {
+                return [key, construct(val, prop, op)[1]];
+            }
+            if (key === '$not') {
+                return [key, construct(val, prop, op)];
+            }
+            if (key[0] === '$') {
+                if (!valid[key])
+                    throw Error(`pgast.invalid_op:${key}`);
+                if (op)
+                    throw Error(`pgast.unexpected_op:${op} before:${key}`);
+                if (!prop)
+                    throw Error(`pgast.expected_prop_before:${key}`);
+                return construct(val, prop, key);
+            }
+            return construct(val, key);
+        }),
+    ];
+}
+function simplify(node) {
+    const op = node[0];
+    // TODO: $and/$or with multiple $subs with same prop ->
+    // single $sub with $and/$or inside
+    // Recurse into subnodes and simplify them first.
+    if (op === '$and' || op === '$or') {
+        node[1] = node[1].map((subnode) => simplify(subnode));
+    }
+    else if (op === '$not') {
+        node[1] = simplify(node[1]);
+    }
+    else if (op === '$sub') {
+        node[2] = simplify(node[2]);
+    }
+    // Handle empty $and/$or and booleans
+    if (op === '$and') {
+        if (!node[1].length)
+            return true;
+        if (node[1].includes(false))
+            return false;
+        node[1] = node[1].filter((item) => item !== true);
+    }
+    else if (op === '$or') {
+        if (!node[1].length)
+            return false;
+        if (node[1].includes(true))
+            return true;
+        node[1] = node[1].filter((item) => item !== false);
+    }
+    else if (op === '$not' && typeof node[1] === 'boolean') {
+        return !node[1];
+    }
+    // $or with multiple $eq limbs with same prop -> $in
+    if (op === '$or') {
+        const { eqmap, noneq, change } = node[1].reduce((acc, item) => {
+            if (item[0] !== '$eq' || item[2] === null) {
+                acc.noneq.push(item);
+            }
+            else if (acc.eqmap[item[1]]) {
+                acc.change = true;
+                acc.eqmap[item[1]].push(item[2]);
+                return acc;
+            }
+            else {
+                acc.eqmap[item[1]] = [item[2]];
+            }
+            return acc;
+        }, { eqmap: {}, noneq: [], change: false });
+        // Don't return. Modify node and then apply the next rule.
+        if (change) {
+            node[1] = [
+                ...noneq,
+                ...Object.entries(eqmap).map(([prop, val]) => val.length > 1 ? ['$in', prop, val] : ['$eq', prop, val[0]]),
+            ];
+        }
+    }
+    // unwrap $and / $or with only one limb
+    if ((op === '$and' || op === '$or') && node[1].length === 1) {
+        return node[1][0];
+    }
+    // $not $eq -> $neq, $in -> $nin etc.
+    if (op === '$not') {
+        const [subop, ...subargs] = node[1];
+        const invop = inverse[subop];
+        return invop ? [invop, ...subargs] : node;
+    }
+    return node;
+}

package/filter/getSql.js

@@ -0,0 +1,100 @@
+import sql, { join, raw } from 'sql-template-tag';
+import { cubeLiteralSql } from "../sql/clauses.js";
+import getAst from "./getAst.js";
+const opSql = {
+    $and: 'AND', // Not SQL as these are used as delimiters
+    $or: 'OR',
+    $not: sql `NOT`,
+    $eq: sql `=`,
+    $neq: sql `<>`,
+    $in: sql `IN`,
+    $nin: sql `NOT IN`,
+    $lt: sql `<`,
+    $lte: sql `<=`,
+    $gt: sql `>`,
+    $gte: sql `>=`,
+    $re: sql `~`,
+    $ire: sql `~*`,
+    $cts: sql `@>`,
+    $ctd: sql `<@`,
+    $keycts: sql `?|`,
+    $keyctd: sql `?&`,
+};
+function getBinarySql(lhs, type, op, value, textLhs) {
+    if (value === null && op === '$eq')
+        return sql `${lhs} IS NULL`;
+    if (value === null && op === '$neq')
+        return sql `${lhs} IS NOT NULL`;
+    const sqlOp = opSql[op];
+    if (!sqlOp)
+        throw Error(`pg.getSql_unknown_operator ${op}`);
+    if (op === '$in' || op === '$nin') {
+        if (type === 'jsonb' && typeof value[0] === 'string')
+            lhs = textLhs;
+        return sql `${lhs} ${sqlOp} (${join(value)})`;
+    }
+    if (op === '$re' || op === '$ire') {
+        if (type === 'jsonb') {
+            lhs = textLhs;
+        }
+        else if (type !== 'text') {
+            lhs = sql `(${lhs})::text`;
+        }
+        return sql `${lhs} ${sqlOp} ${String(value)}`;
+    }
+    if (type === 'jsonb') {
+        if (typeof value === 'string') {
+            return sql `${textLhs} ${sqlOp} ${value}`;
+        }
+        if ((op === '$keycts' || op === '$keyctd') && Array.isArray(value))
+            return sql `${lhs} ${sqlOp} ${value}::text[]`;
+        return sql `${lhs} ${sqlOp} ${JSON.stringify(value)}::jsonb`;
+    }
+    if (type === 'cube')
+        return sql `${lhs} ${sqlOp} ${cubeLiteralSql(value)}`;
+    return sql `${lhs} ${sqlOp} ${value}`;
+}
+function getNodeSql(ast, options) {
+    if (typeof ast === 'boolean')
+        return ast;
+    const op = ast[0];
+    if (op === '$and' || op === '$or') {
+        // Handle variadic operators
+        return sql `(${join(ast[1].map((node) => getNodeSql(node, options)), `) ${opSql[op]} (`)})`;
+    }
+    if (op === '$not') {
+        // Handle unary operators
+        return sql `${opSql[op]} (${getNodeSql(ast[1], options)})`;
+    }
+    if (op === '$sub') {
+        // Handle joins.
+        const joinName = ast[1];
+        if (!options.joins[joinName])
+            throw Error(`pg.no_join ${joinName}`);
+        const { idCol, schema } = options;
+        const joinOptions = options.joins[joinName];
+        const { table: joinTable, refCol } = options.joins[joinName];
+        return sql `"${raw(idCol)}" IN (SELECT "${raw(refCol)}"::${raw(schema.types[idCol])} FROM "${raw(joinTable)}" WHERE ${getNodeSql(ast[2], joinOptions)})`;
+    }
+    // It is a binary operator
+    const [prefix, ...suffix] = ast[1].split('.');
+    const { types = {} } = options.schema;
+    if (!types[prefix])
+        throw Error(`pg.no_column ${prefix}`);
+    if (types[prefix] === 'jsonb') {
+        const [lhs, textLhs] = suffix.length
+            ? [
+                sql `"${raw(prefix)}" #> ${suffix}`,
+                sql `"${raw(prefix)}" #>> ${suffix}`,
+            ]
+            : [sql `"${raw(prefix)}"`, sql `"${raw(prefix)}" #>> '{}'`];
+        return getBinarySql(lhs, 'jsonb', op, ast[2], textLhs);
+    }
+    if (suffix.length)
+        throw Error(`pg.lookup_not_jsonb ${prefix}`);
+    return getBinarySql(sql `"${raw(prefix)}"`, types[prefix], op, ast[2]);
+}
+export default function getSql(filter, options) {
+    const ast = getAst(filter);
+    return getNodeSql(ast, options);
+}

package/filter/index.d.ts

@@ -0,0 +1,2 @@
+export { default as filterObject } from './filterObject.js';
+export { default as getFilterSql } from './getSql.js';

package/index.d.ts

@@ -0,0 +1,8 @@
+/**
+ * @param {Partial<TableOpts> & {connection: any}} options
+ * @returns {Function}
+ */
+export declare const pg: ({ connection, ...rawOptions }: {
+    [x: string]: any;
+    connection: any;
+}) => (store: any) => void;

package/index.js

@@ -0,0 +1,68 @@
+import { encodePath, merge, remove } from '@graffy/common';
+import Db from "./Db.js";
+/**
+ * @typedef {{
+ *    table: string,
+ *    idCol: string,
+ *    verCol: string,
+ *    joins: Record<string, Partial<TableOpts> & { refCol: string }>,
+ *    schema?: any,
+ *    verDefault?: string
+ * }} TableOpts
+ */
+/**
+ * @param {string} name
+ * @param {Partial<TableOpts>} options
+ * @param {string} parentName
+ * @returns {TableOpts}
+ */
+function getTableOpts(name, { table, idCol, verCol, joins, schema, verDefault } = {}, parentName = null) {
+    const tableName = table || name;
+    return {
+        table: table || name,
+        idCol: idCol || 'id',
+        verCol: verCol || 'updatedAt',
+        joins: Object.fromEntries(Object.entries(joins || {}).map(([joinName, { refCol = parentName, ...joinOptions }]) => [
+            joinName,
+            {
+                refCol,
+                ...getTableOpts(joinName, joinOptions, tableName),
+            },
+        ])),
+        schema,
+        verDefault,
+    };
+}
+/**
+ * @param {Partial<TableOpts> & {connection: any}} options
+ * @returns {Function}
+ */
+export const pg = ({ connection, ...rawOptions }) => (store) => {
+    store.on('read', read);
+    store.on('write', write);
+    const prefix = store.path;
+    /** @type {TableOpts & {prefix?: string[]}} */
+    const tableOpts = getTableOpts(prefix[prefix.length - 1], rawOptions);
+    tableOpts.prefix = prefix;
+    const defaultDb = new Db(connection);
+    function read(query, options, next) {
+        const { pgClient } = options;
+        const db = pgClient ? new Db(pgClient) : defaultDb;
+        const readPromise = db.read(query, tableOpts);
+        const remainingQuery = remove(query, encodePath(prefix));
+        const nextPromise = next(remainingQuery);
+        return Promise.all([readPromise, nextPromise]).then(([readRes, nextRes]) => {
+            return merge(readRes, nextRes);
+        });
+    }
+    function write(change, options, next) {
+        const { pgClient } = options;
+        const db = pgClient ? new Db(pgClient) : defaultDb;
+        const writePromise = db.write(change, tableOpts);
+        const remainingChange = remove(change, encodePath(prefix));
+        const nextPromise = next(remainingChange);
+        return Promise.all([writePromise, nextPromise]).then(([writeRes, nextRes]) => {
+            return merge(writeRes, nextRes);
+        });
+    }
+};

package/package.json

@@ -2,23 +2,29 @@
   "name": "@graffy/pg",
   "description": "The standard Postgres module for Graffy. Each instance this module mounts a Postgres table as a Graffy subtree.",
   "author": "aravind (https://github.com/aravindet)",
-  "version": "0.19.0",
-  "main": "./index.cjs",
+  "version": "0.19.1-alpha.1",
+  "main": "./cjs/index.js",
   "exports": {
-    "import": "./index.mjs",
-    "require": "./index.cjs"
+    ".": {
+      "import": "./index.js",
+      "types": "./index.d.ts"
+    },
+    "./*": {
+      "import": "./*.js",
+      "types": "./*.d.ts"
+    }
   },
-  "module": "./index.mjs",
-  "types": "./types/index.d.ts",
+  "types": "./index.d.ts",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/usegraffy/graffy.git"
   },
   "license": "Apache-2.0",
   "dependencies": {
-    "@graffy/common": "0.19.0",
-    "debug": "^4.4.3",
-    "sql-formatter": "^15.7.2"
+    "sql-formatter": "^15.7.2",
+    "@graffy/common": "0.19.1-alpha.1",
+    "sql-template-tag": "^5.2.1",
+    "debug": "^4.4.3"
   },
   "peerDependencies": {
     "pg": "^8.0.0"

package/sql/clauses.d.ts

@@ -0,0 +1,12 @@
+import { Sql } from 'sql-template-tag';
+export declare const getJsonBuildTrusted: (variadic: any) => Sql;
+export declare const lookup: (prop: any, options: any) => Sql;
+export declare const lookupNumeric: (prop: any) => Sql;
+export declare const getSelectCols: (options: any, projection?: any) => Sql;
+export declare function cubeLiteralSql(value: any): Sql;
+export declare const getInsert: (rows: any, options: any) => {
+    cols: Sql;
+    vals: Sql;
+    updates: Sql;
+};
+export declare const getUpdates: (row: any, options: any) => Sql;