@saltcorn/sql 0.5.8 → 0.6.1

package/index.js

@@ -20,8 +20,8 @@ const {
 } = require("@saltcorn/markup/tags");
 const { mkTable } = require("@saltcorn/markup");
 const { readState } = require("@saltcorn/data/plugin-helper");
-
-const _ = require("underscore");
+const { features } = require("@saltcorn/data/db/state");
+const Handlebars = require("handlebars");
 
 const configuration_workflow = () =>
   new Workflow({
@@ -51,7 +51,9 @@ const configuration_workflow = () =>
                 label: "Output type",
                 type: "String",
                 required: true,
-                attributes: { options: ["Table", "JSON", "HTML"] },
+                attributes: {
+                  options: ["Table", "JSON", "HTML", "HTML with handlebars"],
+                },
               },
               {
                 name: "html_code",
@@ -59,14 +61,17 @@ const configuration_workflow = () =>
                 input_type: "code",
                 attributes: { mode: "text/html" },
                 showIf: { output_type: "HTML" },
+                sublabel:
+                  "Use interpolations (<code>{{ }}</code>) to access query result in he <code>rows</code> variable. Example: <code>&lt;script&gt;const rows = {{ JSON.stringify(rows) }}&lt;/script&gt;</code>",
               },
               {
-                input_type: "section_header",
-                label: " ",
-                sublabel: div(
+                name: "html_code",
+                label: "HTML Code",
+                input_type: "code",
+                attributes: { mode: "text/html" },
+                showIf: { output_type: "HTML with handlebars" },
+                sublabel:
                   "Use handlebars to access query result in the <code>rows</code> variable. Example: <code>{{#each rows}}&lt;h1&gt;{{this.name}}&lt;/h1&gt;{{/each}}</code>",
-                ),
-                showIf: { row_count: "Many" },
               },
             ],
           });
@@ -114,11 +119,6 @@ const run = async (
   }
   switch (output_type) {
     case "HTML":
-      /*const template = _.template(html_code || "", {
-        evaluate: /\{\{#(.+?)\}\}/g,
-        interpolate: /\{\{([^#].+?)\}\}/g,
-      });
-      console.log("template", viewname, state, html_code);*/
       return interpolate(
         html_code,
         { rows: qres.rows },
@@ -126,7 +126,9 @@ const run = async (
         `HTML code interpolation in view ${viewname}`,
       );
     //return template();
-
+    case "HTML with handlebars":
+      const template = Handlebars.compile(html_code || "");
+      return template({ rows: qres.rows });
     case "JSON":
       return `<pre>${JSON.stringify(qres.rows, null, 2)}</pre>`;
 
@@ -184,6 +186,29 @@ module.exports = {
       name: "SQLView",
       display_state_form: false,
       tableless: true,
+      copilot_generate_view_prompt: async ({ table }) => {
+        const tableLines = [];
+        const tables = await Table.find({});
+        tables.forEach((table) => {
+          const fieldLines = table.fields.map(
+            (f) =>
+              `  * ${f.name} with type: ${f.pretty_type.replace(
+                "Key to",
+                "ForeignKey referencing",
+              )}.${f.description ? ` ${f.description}` : ""}`,
+          );
+          tableLines.push(
+            `${table.name}${
+              table.description ? `: ${table.description}.` : "."
+            } Contains the following fields:\n${fieldLines.join("\n")}`,
+          );
+        });
+        return `The database already contains the following tables: 
+
+    ${tableLines.join("\n\n")}
+
+    You can reference these tables when generating SQL`;
+      },
       get_state_fields,
       configuration_workflow,
       run,

package/package.json

@@ -1,15 +1,22 @@
 {
   "name": "@saltcorn/sql",
-  "version": "0.5.8",
+  "version": "0.6.1",
   "description": "Actions and views based on SQL",
   "main": "index.js",
   "dependencies": {
     "@saltcorn/markup": "^0.6.0",
     "@saltcorn/data": "^0.6.0",
     "underscore": "1.13.6",
+    "handlebars": "4.7.7",
     "node-sql-parser": "5.3.10",
     "sqlstring": "^2.3.3"
   },
+  "devDependencies": {
+    "jest": "^29.7.0"
+  },
+  "scripts": {
+    "test": "jest tests --runInBand"
+  },
   "author": "Tom Nielsen",
   "license": "MIT",
   "eslintConfig": {
@@ -19,7 +26,8 @@
     },
     "env": {
       "node": true,
-      "es6": true
+      "es6": true,
+      "jest/globals": true
     },
     "rules": {
       "no-unused-vars": "off",

package/table-provider.js

@@ -12,9 +12,27 @@ const { mkTable } = require("@saltcorn/markup");
 const { pre, code } = require("@saltcorn/markup/tags");
 const parser = new Parser();
 const _ = require("underscore");
+const { features } = require("@saltcorn/data/db/state");
+
+const on_create = async (table) => {
+  if (table?.provider_cfg?.sql_view) {
+    await db.query(
+      `CREATE OR REPLACE VIEW "${db.sqlsanitize(table.name)}" AS ${table?.provider_cfg?.sql}`,
+    );
+  }
+};
 
 const configuration_workflow = (req) =>
   new Workflow({
+    onDone: async (ctx) => {
+      const table = Table.findOne(ctx.table_id);
+      if (table && ctx?.sql_view) {
+        await db.query(
+          `CREATE OR REPLACE VIEW "${db.sqlsanitize(table.name)}" AS ${ctx.sql}`,
+        );
+      }
+      return ctx;
+    },
     steps: [
       {
         name: "query",
@@ -46,12 +64,25 @@ const configuration_workflow = (req) =>
                   }
                 },
               },
+              ...(features.table_create_callback
+                ? [
+                    {
+                      label: "Create SQL VIEW",
+                      type: "Bool",
+                      name: "sql_view",
+                      default: true
+                    },
+                  ]
+                : []),
               {
                 label: "Ignore where/order",
                 sublabel:
                   "Always use this SQL directly without attempting to modify it",
                 type: "Bool",
                 name: "ignore_where",
+                showIf: features.table_create_callback
+                  ? { sql_view: false }
+                  : undefined,
               },
             ],
           });
@@ -66,10 +97,10 @@ const configuration_workflow = (req) =>
               label: field.name,
               key: field.name,
             })),
-            qres.rows?.slice?.(0, 5)
+            qres.rows?.slice?.(0, 5),
           );
           const pkey_options = getState().type_names.filter(
-            (tnm) => getState().types[tnm]?.primaryKey
+            (tnm) => getState().types[tnm]?.primaryKey,
           );
           const tables = await Table.find({});
 
@@ -197,7 +228,7 @@ const getSqlQuery = (sql, cfg, where, opts) => {
               as: null,
             }
           : (ast[0].columns || []).find(
-              (c) => k === c.as || (!c.as && k === c.expr?.column)
+              (c) => k === c.as || (!c.as && k === c.expr?.column),
             );
       const sqlExprCol =
         ast[0].columns == "*"
@@ -210,22 +241,22 @@ const getSqlQuery = (sql, cfg, where, opts) => {
       const sqlAggrCol = (ast[0].columns || []).find(
         (c) =>
           c.expr?.type === "aggr_func" &&
-          c.expr?.name?.toUpperCase() === k.toUpperCase()
+          c.expr?.name?.toUpperCase() === k.toUpperCase(),
       );
 
       let left = sqlExprCol
         ? { ...sqlExprCol.expr, as: null }
         : sqlAggrCol
-        ? { ...sqlAggrCol.expr }
-        : {
-            type: "column_ref",
-            table: sqlCol?.expr?.table,
-            column: sqlCol?.expr?.column || db.sqlsanitize(k),
-          };
+          ? { ...sqlAggrCol.expr }
+          : {
+              type: "column_ref",
+              table: sqlCol?.expr?.table,
+              column: sqlCol?.expr?.column || db.sqlsanitize(k),
+            };
       //console.log({ k, sqlCol, sqlExprCol });
       if (!sqlCol) {
         const starCol = (ast[0].columns || []).find(
-          (c) => c.type === "star_ref"
+          (c) => c.type === "star_ref",
         );
         if (starCol)
           left = {
@@ -241,14 +272,14 @@ const getSqlQuery = (sql, cfg, where, opts) => {
           wherek?.ilike && !sqlAggrCol
             ? "ILIKE"
             : wherek?.gt && !sqlAggrCol
-            ? wherek.equal
-              ? ">="
-              : ">"
-            : wherek?.lt && !sqlAggrCol
-            ? wherek.equal
-              ? "<="
-              : "<"
-            : "=",
+              ? wherek.equal
+                ? ">="
+                : ">"
+              : wherek?.lt && !sqlAggrCol
+                ? wherek.equal
+                  ? "<="
+                  : "<"
+                : "=",
         left,
         right:
           wherek?.ilike && !sqlAggrCol
@@ -469,33 +500,59 @@ const countRows = async (cfg, where, opts) => {
   //console.trace({ sqlQ, phValues, opts });
   db.sql_log(
     `select count(*) from (${ensure_no_final_semicolon(sqlQ)})`,
-    phValues
+    phValues,
   );
   const qres = await client.query(
     `select count(*) from (${ensure_no_final_semicolon(sqlQ)})`,
-    phValues
+    phValues,
   );
   qres.query = sqlQ;
   db.sql_log("ROLLBACK;");
   await client.query(`ROLLBACK;`);
 
   if (!is_sqlite) client.release(true);
-  return qres.rows[0].count;
+  return +qres.rows[0].count;
 };
 
 module.exports = {
   "SQL query": {
     configuration_workflow,
     fields: (cfg) => cfg?.columns || [],
-    get_table: (cfg) => {
+    on_create,
+    get_table: (cfg, table) => {
+      let syntheticTable;
+      if (cfg?.sql_view && table)
+        syntheticTable = new Table({
+          ...table,
+          provider_name: undefined,
+          provider_cfg: undefined,
+        });
       return {
+        disableFiltering: true,
         getRows: async (where, opts) => {
-          const qres = await runQuery(cfg, where, opts);
+          if (syntheticTable) return await syntheticTable.getRows(where, opts);
+          const qres = await runQuery(cfg, where || {}, opts || {});
           return qres.rows;
         },
         countRows: async (where, opts) => {
-          return await countRows(cfg, where, opts);
+          if (syntheticTable)
+            return await syntheticTable.countRows(where, opts);
+
+          return await countRows(cfg, where || {}, opts || {});
         },
+        ...(syntheticTable
+          ? {
+              distinctValues: async (fldNm, opts) => {
+                return await syntheticTable.distinctValues(fldNm, opts);
+              },
+              getJoinedRows: async (opts) => {
+                return await syntheticTable.getJoinedRows(opts);
+              },
+              aggregationQuery: async (aggs, opts) => {
+                return await syntheticTable.aggregationQuery(aggs, opts);
+              },
+            }
+          : {}),
       };
     },
   },

package/tests/data.js

@@ -0,0 +1,85 @@
+const sqlusers = {
+  min_role_read: 1,
+  min_role_write: 1,
+  provider_name: "SQL query",
+  provider_cfg: {
+    sql: "select * from users;",
+    columns: [
+      {
+        name: "id",
+        type: "Integer",
+        label: "id",
+        primary_key: true,
+      },
+      {
+        name: "email",
+        type: "String",
+        label: "email",
+      },
+      {
+        name: "password",
+        type: "String",
+        label: "password",
+      },
+      {
+        name: "role_id",
+        type: "Integer",
+        label: "role id",
+      },
+      {
+        name: "reset_password_token",
+        type: "String",
+        label: "reset password token",
+      },
+      {
+        name: "reset_password_expiry",
+        type: "String",
+        label: "reset password expiry",
+      },
+      {
+        name: "language",
+        type: "String",
+        label: "language",
+      },
+      {
+        name: "disabled",
+        type: "Bool",
+        label: "disabled",
+      },
+      {
+        name: "api_token",
+        type: "String",
+        label: "api token",
+      },
+      {
+        name: "_attributes",
+        type: "JSON",
+        label: " attributes",
+      },
+      {
+        name: "verification_token",
+        type: "String",
+        label: "verification token",
+      },
+      {
+        name: "verified_on",
+        type: "String",
+        label: "verified on",
+      },
+      {
+        name: "last_mobile_login",
+        type: "String",
+        label: "last mobile login",
+      },
+    ],
+    //ignore_where: true,
+  },
+  ownership_formula: null,
+};
+
+const sqlusers_view = {
+  ...sqlusers,
+  provider_cfg: { ...sqlusers.provider_cfg, sql_view: true },
+};
+
+module.exports = { sqlusers, sqlusers_view };

package/tests/sqlview.test.js

@@ -0,0 +1,81 @@
+const { getState } = require("@saltcorn/data/db/state");
+const View = require("@saltcorn/data/models/view");
+const Table = require("@saltcorn/data/models/table");
+const Plugin = require("@saltcorn/data/models/plugin");
+
+const { mockReqRes } = require("@saltcorn/data/tests/mocks");
+const { afterAll, beforeAll, describe, it, expect } = require("@jest/globals");
+const db = require("@saltcorn/data/db");
+
+afterAll(require("@saltcorn/data/db").close);
+beforeAll(async () => {
+  await require("@saltcorn/data/db/reset_schema")();
+  await require("@saltcorn/data/db/fixtures")();
+
+  getState().registerPlugin("base", require("@saltcorn/data/base-plugin"));
+  getState().registerPlugin("@saltcorn/sql", require(".."));
+  //db.set_sql_logging(true);
+});
+
+describe("sql view", () => {
+  it("runs", async () => {
+    const view = new View({
+      name: "BookSQLView",
+      description: "",
+      viewtemplate: "SQLView",
+      configuration: {
+        sql: "select id, author, pages from books order by id;",
+        html_code: `<script>const boooks = {{ JSON.stringify(rows) }}<script>`,
+        output_type: "HTML",
+        state_parameters: "",
+      },
+      min_role: 1,
+      table: null,
+    });
+    const result = await view.run({}, mockReqRes);
+
+    expect(result).toBe(
+      '<script>const boooks = [{"id":1,"author":"Herman Melville","pages":967},{"id":2,"author":"Leo Tolstoy","pages":728}]<script>',
+    );
+  });
+  it("run with state params", async () => {
+    const view = new View({
+      name: "BookSQLView",
+      description: "",
+      viewtemplate: "SQLView",
+      configuration: {
+        sql: "select id, author, pages from books where id = $1",
+        html_code: `<script>const boooks = {{ JSON.stringify(rows) }}<script>`,
+        output_type: "HTML",
+        state_parameters: "id",
+      },
+      min_role: 1,
+      table: null,
+    });
+    const result = await view.run({id:2}, mockReqRes);
+
+    expect(result).toBe(
+      '<script>const boooks = [{"id":2,"author":"Leo Tolstoy","pages":728}]<script>',
+    );
+  });
+   it("runs with handlebars", async () => {
+    const view = new View({
+      name: "BookSQLView",
+      description: "",
+      viewtemplate: "SQLView",
+      configuration: {
+        sql: "select id, author, pages from books order by id;",
+        html_code: `{{#each rows}}<h1>{{this.author}}</h1>{{/each}}`,
+        output_type: "HTML with handlebars",
+        state_parameters: "",
+      },
+      min_role: 1,
+      table: null,
+    });
+    const result = await view.run({}, mockReqRes);
+
+    expect(result).toBe(
+      '<h1>Herman Melville</h1><h1>Leo Tolstoy</h1>',
+    );
+  });
+});

package/tests/table-provider.test.js

@@ -0,0 +1,107 @@
+const { getState } = require("@saltcorn/data/db/state");
+const View = require("@saltcorn/data/models/view");
+const Table = require("@saltcorn/data/models/table");
+const Plugin = require("@saltcorn/data/models/plugin");
+
+const { mockReqRes } = require("@saltcorn/data/tests/mocks");
+const { afterAll, beforeAll, describe, it, expect } = require("@jest/globals");
+const db = require("@saltcorn/data/db");
+
+afterAll(require("@saltcorn/data/db").close);
+beforeAll(async () => {
+  await require("@saltcorn/data/db/reset_schema")();
+  await require("@saltcorn/data/db/fixtures")();
+
+  getState().registerPlugin("base", require("@saltcorn/data/base-plugin"));
+  getState().registerPlugin("@saltcorn/sql", require(".."));
+  //db.set_sql_logging(true);
+});
+
+// run with:
+//  saltcorn dev:plugin-test -d ~/sql/
+
+//jest.setTimeout(30000);
+
+describe("sql table provider", () => {
+  it("creates table", async () => {
+    await Table.create("sqlusers", require("./data").sqlusers);
+    await getState().refresh_tables(false);
+  });
+  it("counts table", async () => {
+    const table = Table.findOne("sqlusers");
+    const nus = await table.countRows({});
+    expect(nus).toBe(3);
+    const nadmin = await table.countRows({ role_id: 1 });
+    expect(nadmin).toBe(1);
+  });
+  it("gets rows from table", async () => {
+    const table = Table.findOne("sqlusers");
+    const us = await table.getRows({}, { orderBy: "id" });
+    expect(us.length).toBe(3);
+    expect(us[0].id).toBe(1);
+
+    const twous = await table.getRows({}, { limit: 2 });
+    expect(twous.length).toBe(2);
+    const twous1 = await table.getRows(
+      {},
+      { limit: 2, orderBy: "id", orderDesc: true },
+    );
+    expect(twous1.length).toBe(2);
+    expect(twous1[0].id).toBe(3);
+
+    const admins = await table.getRows({ role_id: 1 });
+    expect(admins.length).toBe(1);
+    expect(admins[0].email).toBe("admin@foo.com");
+    const admin = await table.getRow({ role_id: 1 });
+    expect(admin.email).toBe("admin@foo.com");
+  });
+});
+describe("view-based sql table provider", () => {
+  it("creates table", async () => {
+    await Table.create("sqlusersv", require("./data").sqlusers_view);
+    await getState().refresh_tables(false);
+  });
+  it("counts table", async () => {
+    const table = Table.findOne("sqlusersv");
+    const nus = await table.countRows({});
+    expect(nus).toBe(3);
+    const nadmin = await table.countRows({ role_id: 1 });
+    expect(nadmin).toBe(1);
+  });
+  it("gets rows from table", async () => {
+    const table = Table.findOne("sqlusersv");
+    const us = await table.getRows({}, { orderBy: "id" });
+    expect(us.length).toBe(3);
+    expect(us[0].id).toBe(1);
+
+    const twous = await table.getRows({}, { limit: 2 });
+    expect(twous.length).toBe(2);
+    const twous1 = await table.getRows(
+      {},
+      { limit: 2, orderBy: "id", orderDesc: true },
+    );
+    expect(twous1.length).toBe(2);
+    expect(twous1[0].id).toBe(3);
+
+    const admins = await table.getRows({ role_id: 1 });
+    expect(admins.length).toBe(1);
+    expect(admins[0].email).toBe("admin@foo.com");
+    const admin = await table.getRow({ role_id: 1 });
+    expect(admin.email).toBe("admin@foo.com");
+  });
+  it("has distinct values", async () => {
+    const table = Table.findOne("sqlusersv");
+    const vs = await table.distinctValues("role_id");
+    expect(vs.length).toBe(3);
+    expect(vs).toContain(80);
+  });
+  it("gets joined values", async () => {
+    const table = Table.findOne("sqlusersv");
+    const vs = await table.getJoinedRows({});
+    expect(vs.length).toBe(3);
+    const vs2 = await table.getJoinedRows({ limit: 2 });
+    expect(vs2.length).toBe(2);
+    const admins = await table.getJoinedRows({ where: { role_id: 1 } });
+    expect(admins.length).toBe(1);
+  });
+});