@dwtechs/antity-pgsql 0.13.0 → 0.15.0

package/README.md

@@ -120,6 +120,7 @@ router.get("/", ..., entity.get);
 // Using substacks (recommended) - combines normalize, validate, and database operation
 router.post("/", ...entity.addArraySubstack);
 router.put("/", ...entity.updateArraySubstack);
+router.put("/preferences", ...entity.syncArraySubstack);
 
 // Or manually chain middlewares
 router.post("/manual", entity.normalizeArray, entity.validateArray, ..., entity.add);
@@ -190,6 +191,9 @@ class SQLEntity {
   get addOneSubstack(): SubstackTuple;
   get updateArraySubstack(): SubstackTuple;
   get updateOneSubstack(): SubstackTuple;
+  get upsertArraySubstack(): SubstackTuple;
+  get upsertOneSubstack(): SubstackTuple;
+  get syncArraySubstack(): SubstackTuple;
 
   query: {
     select: (
@@ -223,6 +227,15 @@ class SQLEntity {
         query: string;
           args: unknown[];
       };
+    upsert: (
+      rows: Record<string, unknown>[],
+      conflictTarget: string | string[],
+      consumerId?: number | string,
+      consumerName?: string,
+      rtn?: string) => {
+        query: string;
+        args: unknown[];
+      };
     delete: (ids: number[]) => {
       query: string;
       args: number[];
@@ -233,6 +246,8 @@ class SQLEntity {
   get: (req: Request, res: Response, next: NextFunction) => void;
   add: (req: Request, res: Response, next: NextFunction) => Promise<void>;
   update: (req: Request, res: Response, next: NextFunction) => Promise<void>;
+  upsert: (req: Request, res: Response, next: NextFunction) => Promise<void>;
+  sync: (req: Request, res: Response, next: NextFunction) => Promise<void>;
   archive: (req: Request, res: Response, next: NextFunction) => Promise<void>;
   delete: (req: Request, res: Response, next: NextFunction) => Promise<void>;
   deleteArchive: (req: Request, res: Response, next: NextFunction) => void;
@@ -259,9 +274,13 @@ function execute(
 
 ### Middleware Methods for Express.js
 
-get(), add(), update(), archive(), delete(), deleteArchive() and getHistory() methods are made to be used as Express.js middlewares.
+get(), add(), update(), upsert(), sync(), archive(), delete(), deleteArchive() and getHistory() methods are made to be used as Express.js middlewares.
 Each method will look for data to work on in the **req.body.rows** parameter.
 
+The upsert() method additionally requires **req.body.conflictTarget** to specify which column(s) define uniqueness.
+
+The sync() method accepts an optional **req.body.idField** (defaults to `'id'`) and optional **req.body.filters** to scope which existing rows are considered part of the managed set.
+
 ### Schema Qualification
 
 All SQL queries generated by Antity-pgsql use schema-qualified table names (e.g., `schema.table`). This provides:
@@ -279,17 +298,183 @@ Substacks are pre-composed middleware chains that combine normalization, validat
 - **addOneSubstack**: Combines `normalizeOne`, `validateOne`, and `add`. Use this for POST routes with `req.body` containing a single object.
 - **updateArraySubstack**: Combines `normalizeArray`, `validateArray`, and `update`. Use this for PUT routes with `req.body.rows` containing multiple objects.
 - **updateOneSubstack**: Combines `normalizeOne`, `validateOne`, and `update`. Use this for PUT routes with `req.body` containing a single object.
+- **upsertArraySubstack**: Combines `normalizeArray`, `validateArray`, and `upsert`. Use this for upsert routes with `req.body.rows` containing multiple objects. Requires `req.body.conflictTarget`.
+- **upsertOneSubstack**: Combines `normalizeOne`, `validateOne`, and `upsert`. Use this for upsert routes with `req.body` containing a single object. Requires `req.body.conflictTarget`.
+- **syncArraySubstack**: Combines `normalizeArray`, `validateArray`, and `sync`. Use this for bulk-sync routes with `req.body.rows` containing the full desired state. Rows are inserted, updated, or deleted as needed. Accepts optional `req.body.idField` and `req.body.filters`.
 
 Using substacks simplifies your route definitions and ensures consistent data processing.
 
 ### Query Methods
 
 - **query.select()**: Generates a SELECT query. When the `rows` parameter is provided (not null), pagination is automatically enabled and the query includes `COUNT(*) OVER () AS total` to return the total number of rows. The total count is extracted from results and returned separately from the row data.
+- **query.insert()**: Generates an INSERT query. Accepts an array of objects with properties matching the entity definition. Optionally appends `consumerId` and `consumerName` for history tracking. Supports `RETURNING` clause via the `rtn` parameter.
+- **query.update()**: Generates an UPDATE query using CASE statements. Accepts an array of objects with `id` property. Optionally appends `consumerId` and `consumerName` for history tracking.
+- **query.upsert()**: Generates an INSERT ... ON CONFLICT ... DO UPDATE query. (See [Upsert](#upsert-insert-or-update) section below.) Accepts an array of objects and a `conflictTarget` (single column name or array of column names) that defines uniqueness. If a conflict occurs on the specified column(s), the row is updated; otherwise, it is inserted. Properties are automatically included if they have both INSERT and UPDATE operations. Optionally appends `consumerId` and `consumerName` for history tracking. Supports `RETURNING` clause via the `rtn` parameter.
 - **query.archive()**: Generates a simplified `UPDATE ... SET archived = true WHERE id IN (...)` query. Accepts an array of objects with `id` property. Optionally appends `consumerId` and `consumerName` for history tracking. Does not require an `archived` field in the rows — it is set directly in the SQL.
+- **sync()**: Atomically synchronises the table with the provided rows inside a single PostgreSQL transaction. Missing rows are inserted, existing rows are updated, and rows absent from the list are deleted. Accepts optional `idField` (default `'id'`) and `filters` to restrict the scope of managed rows. Stores the result in `res.locals.rows` and a summary `{ inserted, updated, deleted }` in `res.locals.sync`.
 - **delete()**: Deletes rows by their IDs. Expects `req.body.rows` to be an array of objects with `id` property: `[{id: 1}, {id: 2}]`
 - **deleteArchive()**: Deletes archived rows that were archived before a specific date using a PostgreSQL SECURITY DEFINER function. Expects `req.body.date` to be a Date object.
 - **getHistory()**: Retrieves modification history for rows from the `log.history` table. Expects `req.body.rows` to be an array of objects with `id` property. Returns all historical records for the specified entity IDs.
 
+### Bulk Sync
+
+The sync functionality atomically replaces the managed set of rows in a table with the supplied list. It combines insert, update, and delete in a single PostgreSQL **transaction** — either all changes succeed or none do.
+
+#### How It Works
+
+1. **Fetch existing IDs**: A `SELECT id FROM table` is issued, optionally scoped by `filters`.
+2. **Diff**: Incoming rows without an ID (or with an unknown ID) are inserted; rows with a known ID are updated; existing IDs absent from the incoming list are deleted — all within the same filter scope.
+3. **Transaction**: All three operations run inside `BEGIN` / `COMMIT`. A failure at any step triggers `ROLLBACK`.
+4. **Result**: `res.locals.rows` contains the full synced list (with generated IDs filled in for inserts). `res.locals.sync` contains `{ inserted, updated, deleted }` counts.
+
+#### Usage Examples
+
+**Using the middleware:**
+
+```javascript
+// Route definition
+router.put('/users/sync', ...entity.syncArraySubstack);
+
+// Request body — send the entire desired state
+{
+  rows: [
+    { id: 1, name: 'John Updated', email: 'john@example.com', age: 31 }, // update
+    { name: 'Jane New', email: 'jane@example.com', age: 25 }             // insert
+    // id: 2 is absent → will be deleted
+  ],
+  idField: 'id' // optional, defaults to 'id'
+}
+```
+
+**Scoping with filters (only manage a subset of rows):**
+
+```javascript
+// Only sync rows where age >= 18 — rows outside this filter are left untouched
+{
+  rows: [
+    { id: 1, name: 'John', email: 'john@example.com', age: 30 }
+  ],
+  filters: {
+    age: { value: 18, matchMode: 'gte' }
+  }
+}
+```
+
+**Response locals after sync:**
+
+```javascript
+res.locals.rows  // full list of synced rows (inserts have their new id)
+res.locals.sync  // { inserted: 1, updated: 1, deleted: 1 }
+```
+
+#### Important Notes
+
+- **Atomic**: All insert / update / delete operations are wrapped in a single transaction.
+- **Filter scope**: When `filters` are provided, only rows matching the filter are considered "managed". Rows outside the filter are never touched.
+- **Property selection**: Insert uses `INSERT` properties; update uses `UPDATE` properties — same as the standalone `add` and `update` middlewares.
+- **Consumer tracking**: `consumerId` and `consumerName` from `res.locals` are forwarded to inserts and updates for history tracking.
+
+### Upsert (Insert or Update)
+
+The upsert functionality uses PostgreSQL's `INSERT ... ON CONFLICT ... DO UPDATE` syntax to insert rows or update them if they already exist based on a unique constraint.
+
+#### How It Works
+
+1. **Conflict Target**: You specify which column(s) define uniqueness (e.g., `'id'`, `'email'`, or `['name', 'email']`)
+2. **Property Selection**: Properties are automatically included if they have **both** `INSERT` and `UPDATE` in their `operations` array
+3. **On Conflict**: When a conflict occurs, all columns except the conflict target are updated
+
+#### Usage Examples
+
+**Using the middleware with a single conflict target:**
+
+```javascript
+// Route definition
+router.post('/users/upsert', ...entity.upsertArraySubstack);
+
+// Request body
+{
+  rows: [
+    { id: 1, name: 'John Updated', email: 'john@example.com' },
+    { name: 'Jane New', email: 'jane@example.com' }
+  ],
+  conflictTarget: 'id'
+}
+```
+
+**Using email as conflict target:**
+
+```javascript
+// If a user with this email exists, update their name; otherwise, insert
+{
+  rows: [
+    { name: 'John', email: 'john@example.com', age: 30 }
+  ],
+  conflictTarget: 'email'
+}
+```
+
+**Using multiple columns as conflict target:**
+
+```javascript
+// Unique constraint on combination of name and email
+{
+  rows: [
+    { name: 'John', email: 'john@example.com', age: 30 }
+  ],
+  conflictTarget: ['name', 'email']
+}
+```
+
+**Using the query generator directly:**
+
+```javascript
+const { query, args } = entity.query.upsert(
+  [{ id: 1, name: 'John', email: 'john@example.com' }],
+  'id',
+  1, // consumerId (optional)
+  'admin', // consumerName (optional)
+  'RETURNING id' // return clause (optional)
+);
+// Generates:
+// INSERT INTO public.users (name, email, "consumerId", "consumerName")
+// VALUES ($1, $2, $3, $4)
+// ON CONFLICT (id) DO UPDATE SET 
+//   name = EXCLUDED.name,
+//   email = EXCLUDED.email,
+//   "consumerId" = EXCLUDED."consumerId",
+//   "consumerName" = EXCLUDED."consumerName"
+// RETURNING id
+```
+
+#### Property Configuration for Upsert
+
+Properties are automatically included in upsert if they have both INSERT and UPDATE operations:
+
+```javascript
+{
+  key: 'name',
+  operations: ['SELECT', 'INSERT', 'UPDATE'] // Included in upsert
+}
+
+{
+  key: 'id',
+  operations: ['SELECT', 'UPDATE'] // NOT included (no INSERT)
+}
+
+{
+  key: 'createdAt',
+  operations: ['SELECT', 'INSERT'] // NOT included (no UPDATE)
+}
+```
+
+#### Important Notes
+
+- **Conflict Target Required**: The `conflictTarget` parameter must specify an existing unique constraint or primary key
+- **Mixed Rows**: You can upsert rows with and without IDs in the same request if your conflict target handles it (e.g., using `SERIAL` primary key)
+- **Atomic Operation**: Unlike separate insert/update calls, upsert is a single atomic database operation
+- **Concurrent Safety**: Prevents race conditions when multiple requests try to create the same record
+
 ### Filters
 
 Filters support two formats for maximum flexibility:

package/dist/antity-pgsql.d.ts

@@ -98,6 +98,7 @@ export declare class SQLEntity extends Entity {
   private sel: unknown;
   private ins: unknown;
   private upd: unknown;
+  private ups: unknown;
   private arc: unknown;
   
   constructor(name: string, properties: Property[], schema?: string);
@@ -114,6 +115,9 @@ export declare class SQLEntity extends Entity {
   get addOneSubstack(): SubstackTuple;
   get updateArraySubstack(): SubstackTuple;
   get updateOneSubstack(): SubstackTuple;
+  get upsertArraySubstack(): SubstackTuple;
+  get upsertOneSubstack(): SubstackTuple;
+  get syncArraySubstack(): SubstackTuple;
   
   query: {
     select: (
@@ -155,6 +159,17 @@ export declare class SQLEntity extends Entity {
       args: unknown[];
     };
     
+    upsert: (
+      rows: Record<string, unknown>[],
+      conflictTarget: string | string[],
+      consumerId?: number | string,
+      consumerName?: string,
+      rtn?: string
+    ) => {
+      query: string;
+      args: unknown[];
+    };
+    
     delete: (ids: number[]) => {
       query: string;
       args: number[];
@@ -168,6 +183,8 @@ export declare class SQLEntity extends Entity {
   get(req: Request, res: Response, next: NextFunction): void;
   add(req: Request, res: Response, next: NextFunction): Promise<void>;
   update(req: Request, res: Response, next: NextFunction): Promise<void>;
+  upsert(req: Request, res: Response, next: NextFunction): Promise<void>;
+  sync(req: Request, res: Response, next: NextFunction): Promise<void>;
   archive(req: Request, res: Response, next: NextFunction): Promise<void>;
   delete(req: Request, res: Response, next: NextFunction): Promise<void>;
   deleteArchive(req: Request, res: Response, next: NextFunction): void;

package/dist/antity-pgsql.js

@@ -372,6 +372,75 @@ class Update {
     }
 }
 
+class Upsert {
+    constructor() {
+        this._props = [];
+        this._quotedProps = [];
+        this._nbProps = 0;
+        this._cols = "";
+    }
+    addProp(prop) {
+        this._props.push(prop);
+        this._quotedProps.push(quoteIfUppercase(prop));
+        this._nbProps++;
+        this._cols = this._quotedProps.join(", ");
+    }
+    query(schema, table, rows, conflictTarget, consumerId, consumerName, rtn = "") {
+        if (!conflictTarget ||
+            (Array.isArray(conflictTarget) && conflictTarget.length === 0) ||
+            (typeof conflictTarget === 'string' && conflictTarget.trim() === '')) {
+            throw new Error('conflictTarget must be provided for upsert operation');
+        }
+        const propsToUse = [...this._props];
+        const quotedPropsToUse = [...this._quotedProps];
+        let nbProps = this._nbProps;
+        let cols = this._cols;
+        if (consumerId !== undefined && consumerName !== undefined) {
+            propsToUse.push("consumerId", "consumerName");
+            quotedPropsToUse.push(`"consumerId"`, `"consumerName"`);
+            nbProps += 2;
+            cols += `, "consumerId", "consumerName"`;
+        }
+        const conflictColumns = Array.isArray(conflictTarget)
+            ? conflictTarget.map(col => quoteIfUppercase(col)).join(", ")
+            : quoteIfUppercase(conflictTarget);
+        let query = `INSERT INTO ${quoteIfUppercase(schema)}.${quoteIfUppercase(table)} (${cols}) VALUES `;
+        const args = [];
+        let i = 0;
+        for (const row of rows) {
+            if (consumerId !== undefined && consumerName !== undefined) {
+                row.consumerId = consumerId;
+                row.consumerName = consumerName;
+            }
+            query += `${$i(nbProps, i)}, `;
+            for (const prop of propsToUse) {
+                args.push(row[prop]);
+            }
+            i += nbProps;
+        }
+        query = query.slice(0, -2);
+        query += ` ON CONFLICT (${conflictColumns}) DO UPDATE SET `;
+        const conflictTargetArray = Array.isArray(conflictTarget) ? conflictTarget : [conflictTarget];
+        const updateCols = quotedPropsToUse.filter((_, idx) => {
+            const propName = propsToUse[idx];
+            return !conflictTargetArray.includes(propName);
+        });
+        for (const col of updateCols) {
+            query += `${col} = EXCLUDED.${col}, `;
+        }
+        query = query.slice(0, -2);
+        if (rtn)
+            query += ` ${rtn}`;
+        return { query, args };
+    }
+    rtn(prop) {
+        return `RETURNING ${quoteIfUppercase(prop)}`;
+    }
+    execute(query, args, client) {
+        return execute(query, args, client);
+    }
+}
+
 var __awaiter$2 = (undefined && undefined.__awaiter) || function (thisArg, _arguments, P, generator) {
     function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {
@@ -613,6 +682,7 @@ class SQLEntity extends Entity {
         this.sel = new Select();
         this.ins = new Insert();
         this.upd = new Update();
+        this.ups = new Upsert();
         this.arc = new Archive();
         this.query = {
             select: (first = 0, rows = null, sortField = null, sortOrder = null, filters = null) => {
@@ -624,6 +694,9 @@ class SQLEntity extends Entity {
             insert: (rows, consumerId, consumerName, rtn = "") => {
                 return this.ins.query(this.schema, this.table, rows, consumerId, consumerName, rtn);
             },
+            upsert: (rows, conflictTarget, consumerId, consumerName, rtn = "") => {
+                return this.ups.query(this.schema, this.table, rows, conflictTarget, consumerId, consumerName, rtn);
+            },
             delete: (ids) => {
                 return queryById(this.schema, this.table, ids);
             },
@@ -706,6 +779,39 @@ class SQLEntity extends Entity {
             l.rows = r;
             next();
         });
+        this.upsert = (req, res, next) => __awaiter(this, void 0, void 0, function* () {
+            const l = res.locals;
+            const rows = req.body.rows;
+            const conflictTarget = req.body.conflictTarget;
+            const dbClient = l.dbClient || null;
+            const cId = l.consumerId;
+            const cName = l.consumerName;
+            if (!conflictTarget) {
+                return next({ status: 400, msg: "Missing conflictTarget for upsert operation" });
+            }
+            if (!rows || !Array.isArray(rows) || rows.length === 0) {
+                return next({ status: 400, msg: "Missing or empty rows array for upsert operation" });
+            }
+            log.debug(`${LOGS_PREFIX}upsert(rows=${rows.length}, conflictTarget=${conflictTarget}, consumerId=${cId})`);
+            const rtn = this.ups.rtn("id");
+            const chunks = chunk(rows);
+            for (const c of chunks) {
+                const { query, args } = this.ups.query(this._schema, this._table, c, conflictTarget, cId, cName, rtn);
+                let db;
+                try {
+                    db = yield execute(query, args, dbClient);
+                }
+                catch (err) {
+                    return next(err);
+                }
+                const r = db.rows;
+                for (let i = 0; i < c.length; i++) {
+                    c[i].id = r[i].id;
+                }
+            }
+            l.rows = flatten(chunks);
+            next();
+        });
         this.archive = (req, res, next) => __awaiter(this, void 0, void 0, function* () {
             const l = res.locals;
             let r = req.body.rows;
@@ -774,6 +880,74 @@ class SQLEntity extends Entity {
             })
                 .catch((err) => next(err));
         };
+        this.sync = (req, res, next) => __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            const l = res.locals;
+            const rows = req.body.rows;
+            const idField = (_a = req.body.idField) !== null && _a !== void 0 ? _a : 'id';
+            const cId = l.consumerId;
+            const cName = l.consumerName;
+            if (!rows || !Array.isArray(rows)) {
+                return next({ status: 400, msg: "Missing or invalid rows array for sync operation" });
+            }
+            log.debug(`${LOGS_PREFIX}sync(rows=${rows.length}, idField=${idField}, consumerId=${cId})`);
+            const cleanedFilters = cleanFilters(req.body.filters, this.properties) || null;
+            const { conditions, args: filterArgs } = add(cleanedFilters);
+            const whereClause = conditions.length ? ` WHERE ${conditions.join(' AND ')}` : '';
+            const txClient = l.dbClient || (yield pool.connect());
+            let toInsert = [];
+            let toUpdate = [];
+            let idsToDelete = [];
+            try {
+                yield txClient.query('BEGIN');
+                const selectIdQuery = `SELECT ${quoteIfUppercase(idField)} FROM ${quoteIfUppercase(this._schema)}.${quoteIfUppercase(this._table)}${whereClause}`;
+                const existingDb = yield execute(selectIdQuery, filterArgs, txClient);
+                const existingIds = new Set(existingDb.rows.map(r => r[idField]));
+                const incomingIds = new Set(rows.filter(r => r[idField] != null).map(r => r[idField]));
+                toInsert = rows.filter(r => r[idField] == null || !existingIds.has(r[idField]));
+                toUpdate = rows.filter(r => r[idField] != null && existingIds.has(r[idField]));
+                idsToDelete = [...existingIds].filter(id => !incomingIds.has(id));
+                if (toInsert.length > 0) {
+                    const rtn = this.ins.rtn(idField);
+                    const chunks = chunk(toInsert);
+                    for (const c of chunks) {
+                        const { query, args } = this.ins.query(this._schema, this._table, c, cId, cName, rtn);
+                        const db = yield execute(query, args, txClient);
+                        const r = db.rows;
+                        for (let i = 0; i < c.length; i++) {
+                            c[i][idField] = r[i][idField];
+                        }
+                    }
+                }
+                if (toUpdate.length > 0) {
+                    const chunks = chunk(toUpdate);
+                    for (const c of chunks) {
+                        const { query, args } = this.upd.query(this._schema, this._table, c, cId, cName);
+                        yield execute(query, args, txClient);
+                    }
+                }
+                if (idsToDelete.length > 0) {
+                    const deleteArgs = [idsToDelete, ...filterArgs];
+                    const scopedWhere = conditions.length
+                        ? ` AND ${conditions.map(c => c.replace(/\$(\d+)/g, (_, n) => `$${parseInt(n) + 1}`)).join(' AND ')}`
+                        : '';
+                    const deleteQuery = `DELETE FROM ${quoteIfUppercase(this._schema)}.${quoteIfUppercase(this._table)} WHERE ${quoteIfUppercase(idField)} = ANY($1)${scopedWhere}`;
+                    yield execute(deleteQuery, deleteArgs, txClient);
+                }
+                yield txClient.query('COMMIT');
+            }
+            catch (err) {
+                yield txClient.query('ROLLBACK');
+                return next(err);
+            }
+            finally {
+                if (!l.dbClient)
+                    txClient.release();
+            }
+            l.rows = rows;
+            l.sync = { inserted: toInsert.length, updated: toUpdate.length, deleted: idsToDelete.length };
+            next();
+        });
         this._table = name;
         this._schema = schema;
         log.info(`${LOGS_PREFIX}Creating SQLEntity: "${name}"`);
@@ -813,7 +987,18 @@ class SQLEntity extends Entity {
     get updateOneSubstack() {
         return [this.normalizeOne, this.validateOne, this.update];
     }
+    get upsertArraySubstack() {
+        return [this.normalizeArray, this.validateArray, this.upsert];
+    }
+    get upsertOneSubstack() {
+        return [this.normalizeOne, this.validateOne, this.upsert];
+    }
+    get syncArraySubstack() {
+        return [this.normalizeArray, this.validateArray, this.sync];
+    }
     mapProps(operations, key) {
+        let hasInsert = false;
+        let hasUpdate = false;
         for (const o of operations) {
             switch (o) {
                 case "SELECT":
@@ -821,12 +1006,17 @@ class SQLEntity extends Entity {
                     break;
                 case "UPDATE":
                     this.upd.addProp(key);
+                    hasUpdate = true;
                     break;
                 case "INSERT":
                     this.ins.addProp(key);
+                    hasInsert = true;
                     break;
             }
         }
+        if (hasInsert && hasUpdate) {
+            this.ups.addProp(key);
+        }
     }
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dwtechs/antity-pgsql",
-  "version": "0.13.0",
+  "version": "0.15.0",
   "description": "Open source library to add PostgreSQL support to @dwtechs/Antity entities.",
   "keywords": [
     "entities"
@@ -40,7 +40,7 @@
     "@dwtechs/winstan": "0.5.0",
     "@dwtechs/antity": "0.16.0",
     "@dwtechs/sparray": "0.2.1",
-    "pg": "8.13.1"
+    "pg": "8.20.0"
   },
   "devDependencies": {
     "@babel/core": "7.26.0",