@things-factory/warehouse-base 4.3.686 → 4.3.689

package/server/service/inventory-history/inventory-history-query.ts

@@ -1700,253 +1700,211 @@ async function massageInventoryPalletSummary(tx: EntityManager, params: ListPara
   let toDate = params.filters.find(data => data.name === 'toDate')
   let hasTransactionOrBalanceFilter = params.filters.find(data => data.name === 'hasTransactionOrBalance')
 
-  let hasTransactionOrBalanceQuery = ''
+  let hasTransactionOrBalanceQuery = 'TRUE'
   if (hasTransactionOrBalanceFilter && hasTransactionOrBalanceFilter.value) {
-    hasTransactionOrBalanceQuery = 'and (c.opening_qty > 0 or c.total_in_qty > 0 or c.adjustment_qty > 0)'
+    hasTransactionOrBalanceQuery = '(merged.opening_qty > 0 or merged.total_in_qty > 0 or merged.adjustment_qty > 0)'
   }
 
+  // Build a slimmer history table for pallet summary to minimize scans
   await tx.query(
     `
-   create temp table temp_inventory_pallet_summary ON COMMIT DROP as
-    with params as (
-      select $1::timestamp as t1, $2::timestamp as t2
-    ),
-    base as (
-      select
-        invh.*,
-        prd.sku         as product_sku,
-        prd.name        as product_name,
-        prd.description as product_description,
-        prd.type        as product_type
-      from temp_inv_history invh
-      join temp_products prd on prd.id = invh.product_id
-    ),
-
-    /* ---- Opening snapshot as-of t1 (FIX) ----
-      For each pallet, take the last row <= t1; if status='STORED',
-      count it toward that row's product_id opening.
-    */
-    opening_last_asof as (
-      select *
-      from (
-        select h.*,
-              row_number() over (partition by h.pallet_id order by h.created_at desc, h.seq desc) as rn
-        from temp_inv_history h
-        join params p on true
-        where h.created_at <= p.t1
-      ) x
-      where rn = 1 and status = 'STORED'
-    ),
-    opening_agg as (
-      select
-        p.sku  as product_sku,
-        p.id   as product_id,
-        p.name as product_name,
-        p.description as product_description,
-        p.type as product_type,
-        count(*)::int as opening_qty
-      from opening_last_asof o
-      join temp_products p on p.id = o.product_id
-      group by p.sku, p.id, p.name, p.description, p.type
-    ),
-
-    /* ---- Arrivals inside window: first arrival-like row per pallet (your rule) ---- */
-    invIn as (
-      select b.*,
-            row_number() over (partition by b.pallet_id order by b.created_at, b.seq) as rn
-      from base b
-      join params p on true
-      where b.transaction_type in ('UNLOADING','NEW','CANCEL_ORDER')
-        and b.created_at >= p.t1
-    ),
-    invIn_first as (
-      select
-        pallet_id, seq, status, transaction_type,
-        product_sku, product_id, product_name, product_description, product_type,
-        inventory_history_id, packing_type, packing_size, qty, opening_qty, created_at
-      from invIn
-      where rn = 1
-    ),
-
-    /* ---- TERMINATED reps (incl. ADJUSTMENT terminations), using your runs logic ---- */
-    status_runs as (
-      select
-        endData.pallet_id,
-        min(endData.created_at) as started_at,
-        max(endData.created_at) as ended_at,
-        min(endData.seq)        as min_seq,
-        max(endData.seq)        as max_seq,
-        endData.status
-      from (
-        select startData.*,
-              sum(startflag) over (partition by startData.pallet_id order by startData.seq) as grp
-        from (
-          select s.*,
-                case when lag(s.status) over (partition by s.pallet_id order by s.seq) = s.status then 0 else 1 end as startflag
-          from temp_inv_history s
-        ) startData
-      ) endData
-      group by endData.pallet_id, endData.status, endData.grp
-    ),
-    invOut_base as (
-      select b.*, r.started_at
-      from base b
-      join status_runs r
-        on r.pallet_id = b.pallet_id
-      and r.min_seq   <= b.seq
-      and r.max_seq   >= b.seq
-      where b.status = 'TERMINATED'
-        and b.transaction_type in ('TERMINATED','ADJUSTMENT')
-    ),
-
-    /* ---- Transfer pairing (seq-gap tolerant & tolerant txn tags) ---- */
-    /* STORED side in-window; pair to nearest prior TERMINATED (from raw history), different product */
-    adj_stored AS (
-      SELECT b.*
-      FROM base b
-      JOIN params p ON TRUE
-      WHERE b.status = 'STORED'
-        AND b.created_at BETWEEN p.t1 AND p.t2
-    ),
-    adj_pairs AS (
-      SELECT
-        s.pallet_id,
-        s.seq                  AS stored_seq,
-        t.seq                  AS terminated_seq,
-        s.product_id           AS new_product_id,
-        t.product_id           AS old_product_id,
-        s.inventory_history_id AS stored_inventory_history_id,
-        t.inventory_history_id AS terminated_inventory_history_id,
-        s.created_at           AS stored_at,
-        t.created_at           AS terminated_at,
-        s.transaction_type     AS stored_txn,
-        t.transaction_type     AS terminated_txn
-      FROM adj_stored s
-      JOIN LATERAL (
-        SELECT t.*
-        FROM temp_inv_history t                  -- use raw history (old product may be outside temp_products)
-        WHERE t.pallet_id = s.pallet_id
-          AND t.seq < s.seq
-          AND t.status = 'TERMINATED'
-        ORDER BY t.seq DESC
-        LIMIT 1
-      ) t ON TRUE
-      WHERE t.product_id IS DISTINCT FROM s.product_id
-        AND (s.transaction_type='ADJUSTMENT' OR t.transaction_type='ADJUSTMENT')
-    ),
-    adj_pairs_terminated_keys AS (
-      SELECT pallet_id, terminated_seq AS seq
-      FROM adj_pairs
-    ),
-
-    /* Representative TERMINATED per (pallet, product), EXCLUDING those used by a pair (avoid double-count) */
-    invOut_rep AS (
-      SELECT *
-      FROM (
-        SELECT iob.*,
-              row_number() over (partition by iob.pallet_id, iob.product_id order by iob.seq desc) as rn
-        FROM invOut_base iob
-        LEFT JOIN adj_pairs_terminated_keys k
-          ON k.pallet_id = iob.pallet_id
-        AND k.seq       = iob.seq
-        WHERE k.pallet_id IS NULL
-      ) z
-      WHERE rn = 1
-    ),
-
-    /* Materialize both sides of the paired transfer */
-    adj_pairs_stored_rows AS (
-      SELECT
-        b.pallet_id, b.seq, b.status, b.transaction_type,
-        b.product_sku, b.product_id, b.product_name, b.product_description, b.product_type,
-        b.inventory_history_id, b.packing_type, b.packing_size, b.qty, b.opening_qty, b.created_at
-      FROM base b
-      JOIN adj_pairs ap
-        ON ap.pallet_id = b.pallet_id
-      AND ap.stored_seq = b.seq
-    ),
-    adj_pairs_terminated_rows AS (
-      SELECT
-        b.pallet_id, b.seq, b.status, b.transaction_type,
-        b.product_sku, b.product_id, b.product_name, b.product_description, b.product_type,
-        b.inventory_history_id, b.packing_type, b.packing_size, b.qty, b.opening_qty, b.created_at
-      FROM base b
-      JOIN adj_pairs ap
-        ON ap.pallet_id = b.pallet_id
-      AND ap.terminated_seq = b.seq
-    ),
-
-    /* ---- Unified stream for window movement ---- */
-    invHistory as (
-      select * from invIn_first
-      union all
-      select
-        pallet_id, seq, status, transaction_type,
-        product_sku, product_id, product_name, product_description, product_type,
-        inventory_history_id, packing_type, packing_size, qty, opening_qty, created_at
-      from invOut_rep
-      union all
-      select * from adj_pairs_stored_rows
-      union all
-      select * from adj_pairs_terminated_rows
-    ),
-
-    /* ---- Movement aggregation only (no opening here) ---- */
-    movement_agg as (
-      select
-        ih.product_sku,
-        ih.product_id,
-        ih.product_name,
-        ih.product_description,
-        ih.product_type,
-
-        sum(case when ih.created_at >= (select t1 from params)
-                  and ih.created_at <= (select t2 from params) then
-              case when ih.transaction_type in ('UNLOADING','NEW','CANCEL_ORDER') then 1 else 0 end
-            else 0 end)::int as total_in_qty,
-
-        sum(case when ih.created_at >= (select t1 from params)
-                  and ih.created_at <= (select t2 from params) then
-              case when ih.status = 'TERMINATED'
-                    and ih.transaction_type <> 'ADJUSTMENT' then -1 else 0 end
-            else 0 end)::int as total_out_qty,
-
-        sum(case when ih.created_at >= (select t1 from params)
-                  and ih.created_at <= (select t2 from params) then
-              case when ih.status = 'TERMINATED' and ih.transaction_type = 'ADJUSTMENT' then -1
-                  when ih.status = 'STORED'     and ih.transaction_type = 'ADJUSTMENT' then  1
-                  else 0 end
-            else 0 end)::int as adjustment_qty
-
-      from invHistory ih
-      group by ih.product_sku, ih.product_id, ih.product_name, ih.product_description, ih.product_type
-    ),
-
-    /* ---- Combine opening snapshot + window movement ---- */
-    combined as (
-      select
-        coalesce(o.product_sku, m.product_sku)          as product_sku,
-        coalesce(o.product_id,  m.product_id)           as product_id,
-        coalesce(o.product_name,m.product_name)         as product_name,
-        coalesce(o.product_description,m.product_description) as product_description,
-        coalesce(o.product_type,m.product_type)         as product_type,
-
-        coalesce(o.opening_qty, 0)                      as opening_qty,
-        coalesce(m.total_in_qty, 0)                     as total_in_qty,
-        coalesce(m.total_out_qty, 0)                    as total_out_qty,
-        coalesce(m.adjustment_qty, 0)                   as adjustment_qty
-      from opening_agg o
-      full outer join movement_agg m
-        on m.product_id = o.product_id
-    )
+    create temp table temp_inv_history_ps on commit drop as (
+      select pallet_id, product_id, seq, status, transaction_type, created_at
+      from temp_inv_history
+      where (transaction_type in ('UNLOADING','NEW','CANCEL_ORDER','ADJUSTMENT')
+             or status in ('STORED','TERMINATED'))
+        and created_at <= $1::timestamp
+    );
+    `,
+    [toDate.value]
+  )
+
+  await tx.query(
+    `
+    CREATE INDEX IF NOT EXISTS idx_tihps_created_at ON temp_inv_history_ps (created_at);
+    CREATE INDEX IF NOT EXISTS idx_tihps_pallet_seq ON temp_inv_history_ps (pallet_id, seq);
+    CREATE INDEX IF NOT EXISTS idx_tihps_pallet_product_seq ON temp_inv_history_ps (pallet_id, product_id, seq);
+    CREATE INDEX IF NOT EXISTS idx_tihps_product ON temp_inv_history_ps (product_id);
+    CREATE INDEX IF NOT EXISTS idx_tihps_product_created ON temp_inv_history_ps (product_id, created_at);
+    CREATE INDEX IF NOT EXISTS idx_tihps_status ON temp_inv_history_ps (status);
+    CREATE INDEX IF NOT EXISTS idx_tihps_txn ON temp_inv_history_ps (transaction_type);
+    ANALYZE temp_inv_history_ps;
+    `
+  )
 
-    select
-      c.*,
-      (c.opening_qty + c.total_in_qty + c.total_out_qty + c.adjustment_qty) as closing_qty
-    from combined c
-    where 1=1
-    ${hasTransactionOrBalanceQuery}
-    ;
+  await tx.query(
+    `
+    -- Params:
+    -- $1 = t1 ::timestamp
+    -- $2 = t2 ::timestamp
+
+    CREATE TEMP TABLE temp_inventory_pallet_summary ON COMMIT DROP AS
+    WITH
+      params AS (SELECT $1::timestamp AS t1, $2::timestamp AS t2),
+
+      /* Pre-filtered slices (minimal cols) */
+      rows_le_t1 AS (
+        SELECT pallet_id, product_id, seq, status
+        FROM temp_inv_history_ps
+        WHERE created_at <= (SELECT t1 FROM params)
+      ),
+      rows_win AS (
+        SELECT pallet_id, product_id, seq, status, transaction_type
+        FROM temp_inv_history_ps
+        WHERE created_at BETWEEN (SELECT t1 FROM params) AND (SELECT t2 FROM params)
+      ),
+      rows_le_t2 AS (
+        SELECT pallet_id, product_id, seq, status, transaction_type
+        FROM temp_inv_history_ps
+        WHERE created_at <= (SELECT t2 FROM params)
+      ),
+
+      /* Opening (<= t1): last-by-seq per pallet, must be STORED */
+      opening_last_asof AS (
+        SELECT DISTINCT ON (h.pallet_id) h.pallet_id, h.product_id, h.seq, h.status
+        FROM rows_le_t1 h
+        ORDER BY h.pallet_id, h.seq DESC
+      ),
+      opening_anchor AS (
+        SELECT pallet_id, product_id
+        FROM opening_last_asof
+        WHERE status = 'STORED'
+      ),
+
+      /* Closing (<= t2): last STORED per (pallet, product) by seq + validity */
+      closing_last_stored AS (
+        SELECT DISTINCT ON (h.pallet_id, h.product_id) h.pallet_id, h.product_id, h.seq
+        FROM rows_le_t2 h
+        WHERE h.status = 'STORED'
+        ORDER BY h.pallet_id, h.product_id, h.seq DESC
+      ),
+      last_term_by_pair AS (
+        SELECT DISTINCT ON (h.pallet_id, h.product_id) h.pallet_id, h.product_id, h.seq AS last_term_seq
+        FROM rows_le_t2 h
+        WHERE h.status = 'TERMINATED'
+        ORDER BY h.pallet_id, h.product_id, h.seq DESC
+      ),
+      inbound_after_term AS (
+        SELECT cls.pallet_id, cls.product_id
+        FROM closing_last_stored cls
+        LEFT JOIN last_term_by_pair lt
+          ON lt.pallet_id = cls.pallet_id AND lt.product_id = cls.product_id
+        WHERE EXISTS (
+          SELECT 1
+          FROM rows_le_t2 h
+          WHERE h.pallet_id = cls.pallet_id
+            AND h.product_id = cls.product_id
+            AND h.transaction_type IN ('UNLOADING','NEW','CANCEL_ORDER')
+            AND h.seq <= cls.seq
+            AND (lt.last_term_seq IS NULL OR h.seq > lt.last_term_seq)
+        )
+      ),
+      valid_closing AS (
+        SELECT cls.pallet_id, cls.product_id
+        FROM closing_last_stored cls
+        LEFT JOIN inbound_after_term ia
+          ON ia.pallet_id = cls.pallet_id AND ia.product_id = cls.product_id
+        LEFT JOIN last_term_by_pair lt
+          ON lt.pallet_id = cls.pallet_id AND lt.product_id = cls.product_id
+        WHERE lt.last_term_seq IS NULL OR ia.pallet_id IS NOT NULL
+      ),
+      closing_anchor AS (
+        SELECT pallet_id, product_id FROM valid_closing
+      ),
+
+      /* Movement presence in window */
+      inbound_pairs AS (
+        SELECT pallet_id, product_id
+        FROM rows_win
+        WHERE transaction_type IN ('UNLOADING','NEW','CANCEL_ORDER')
+        GROUP BY pallet_id, product_id
+      ),
+      terminated_pairs AS (
+        SELECT pallet_id, product_id
+        FROM rows_win
+        WHERE status = 'TERMINATED' AND transaction_type <> 'ADJUSTMENT'
+        GROUP BY pallet_id, product_id
+      ),
+      adjustment_last AS (
+        SELECT x.pallet_id, x.product_id,
+              CASE WHEN x.status = 'TERMINATED' THEN -1
+                    WHEN x.status = 'STORED'     THEN  1
+                    ELSE 0 END AS adj_sign
+        FROM (
+          SELECT a.pallet_id, a.product_id, MAX(a.seq) AS max_seq
+          FROM rows_win a
+          WHERE a.transaction_type = 'ADJUSTMENT'
+          GROUP BY a.pallet_id, a.product_id
+        ) s
+        JOIN rows_win x
+          ON x.pallet_id = s.pallet_id AND x.product_id = s.product_id AND x.seq = s.max_seq
+      ),
+
+      /* Union anchor (opening OR any movement OR valid closing) */
+      anchor_union AS (
+        SELECT * FROM opening_anchor
+        UNION ALL
+        SELECT * FROM inbound_pairs
+        UNION ALL
+        SELECT * FROM terminated_pairs
+        UNION ALL
+        SELECT * FROM closing_anchor
+      ),
+      anchor_distinct AS (
+        SELECT DISTINCT pallet_id, product_id FROM anchor_union
+      ),
+
+      /* Opening aggregation (anchored) */
+      opening_agg AS (
+        SELECT ol.product_id,
+              COUNT(*) FILTER (WHERE ol.status = 'STORED')::int AS opening_qty
+        FROM opening_last_asof ol
+        JOIN anchor_distinct au ON au.pallet_id = ol.pallet_id AND au.product_id = ol.product_id
+        GROUP BY ol.product_id
+      ),
+
+      /* Movement aggregation (anchored) */
+      movement_agg AS (
+        SELECT
+          COALESCE(inb.product_id, term.product_id, adj.product_id) AS product_id,
+          SUM((inb.pallet_id IS NOT NULL)::int)  AS total_in_qty,
+          SUM((term.pallet_id IS NOT NULL)::int) AS total_out_qty,
+          COALESCE(SUM(adj.adj_sign), 0)::int AS adjustment_qty
+        FROM anchor_distinct au
+        LEFT JOIN inbound_pairs    inb  ON inb.pallet_id = au.pallet_id AND inb.product_id = au.product_id
+        LEFT JOIN terminated_pairs term ON term.pallet_id = au.pallet_id AND term.product_id = au.product_id
+        LEFT JOIN adjustment_last  adj  ON adj.pallet_id = au.pallet_id AND adj.product_id = au.product_id
+        GROUP BY COALESCE(inb.product_id, term.product_id, adj.product_id)
+      ),
+
+      merged AS (
+        SELECT
+          COALESCE(o.product_id, m.product_id) AS product_id,
+          COALESCE(o.opening_qty, 0)           AS opening_qty,
+          COALESCE(m.total_in_qty, 0)          AS total_in_qty,
+          COALESCE(m.total_out_qty, 0)         AS total_out_qty,
+          COALESCE(m.adjustment_qty, 0)        AS adjustment_qty
+        FROM opening_agg o
+        FULL JOIN movement_agg m ON m.product_id = o.product_id
+      )
+
+    SELECT
+      p.sku         AS product_sku,
+      p.id          AS product_id,
+      p.name        AS product_name,
+      p.description AS product_description,
+      p.type        AS product_type,
+      COALESCE(merged.opening_qty, 0)    AS opening_qty,
+      COALESCE(merged.total_in_qty, 0)   AS total_in_qty,
+      COALESCE(merged.total_out_qty, 0)  AS total_out_qty,
+      COALESCE(merged.adjustment_qty, 0) AS adjustment_qty,
+      (COALESCE(merged.opening_qty, 0)
+      + COALESCE(merged.total_in_qty, 0)
+      + COALESCE(merged.total_out_qty, 0)
+      + COALESCE(merged.adjustment_qty, 0))::int AS closing_qty
+    FROM merged
+    JOIN temp_products p ON p.id = merged.product_id
+    WHERE ${hasTransactionOrBalanceQuery};
     `,
     [fromDate.value, toDate.value]
   )
@@ -2012,6 +1970,13 @@ async function productsQuery(tx: EntityManager, params: ListParam, bizplace: Biz
     )`,
     [bizplace.id]
   )
+
+  await tx.query(
+    `
+    CREATE INDEX IF NOT EXISTS idx_tprod_id ON temp_products (id);
+    ANALYZE temp_products;
+    `
+  )
 }
 
 async function filterInventoryQuery(tx: EntityManager, params: ListParam, bizplace: Bizplace, context: any) {
@@ -2054,12 +2019,24 @@ async function filterInventoryQuery(tx: EntityManager, params: ListParam, bizpla
       WHERE ih.domain_id = $1
       AND ih.bizplace_id = $2
       AND ih.created_at <= $3
-      AND (ih.qty <> 0 OR ih.opening_qty <> 0 OR ih.status = 'MISSING')
+      AND EXISTS (SELECT 1 FROM temp_products prd WHERE prd.id = ih.product_id::uuid)
+      ${batchNoQuery}
     )
     `,
     [context.state.domain.id, bizplace.id, toDate.value]
   )
 
+  // Speed up joins and windowed lookups that follow
+  await tx.query(
+    `
+    CREATE INDEX IF NOT EXISTS idx_tfh_pallet_domain ON temp_filtered_histories (pallet_id, domain_id);
+    CREATE INDEX IF NOT EXISTS idx_tfh_product ON temp_filtered_histories (product_id);
+    CREATE INDEX IF NOT EXISTS idx_tfh_created_at ON temp_filtered_histories (created_at);
+    CREATE INDEX IF NOT EXISTS idx_tfh_pallet_seq ON temp_filtered_histories (pallet_id, seq);
+    ANALYZE temp_filtered_histories;
+    `
+  )
+
   /* Step 2: build temp_inv_history by joining inventories with indexed temp_filtered_histories and temp_products */
   await tx.query(
     `
@@ -2072,8 +2049,21 @@ async function filterInventoryQuery(tx: EntityManager, params: ListParam, bizpla
       AND ih.domain_id = i2.domain_id
       INNER JOIN temp_products prd
       ON prd.id = ih.product_id::uuid
-      ${batchNoQuery}
     )
     `
   )
+
+  // Critical indexes for the heavy pallet summary query
+  await tx.query(
+    `
+    CREATE INDEX IF NOT EXISTS idx_tih_created_at ON temp_inv_history (created_at);
+    CREATE INDEX IF NOT EXISTS idx_tih_pallet_seq ON temp_inv_history (pallet_id, seq);
+    CREATE INDEX IF NOT EXISTS idx_tih_pallet_product_seq ON temp_inv_history (pallet_id, product_id, seq);
+    CREATE INDEX IF NOT EXISTS idx_tih_product ON temp_inv_history (product_id);
+    CREATE INDEX IF NOT EXISTS idx_tih_pallet_product ON temp_inv_history (pallet_id, product_id);
+    CREATE INDEX IF NOT EXISTS idx_tih_status ON temp_inv_history (status);
+    CREATE INDEX IF NOT EXISTS idx_tih_txn ON temp_inv_history (transaction_type);
+    ANALYZE temp_inv_history;
+    `
+  )
 }