@autofleet/sheilta 2.6.0 → 2.7.0-0

package/lib/index.d.ts

@@ -229,7 +229,8 @@ type MessageAck = () => Promise<void>;
 type MessageNack = (err?: Error, requeue?: boolean) => Promise<void>;
 type BulkPerIdHandler<T = any> = (args: {
   jobId?: string;
-  id: Id;
+  ids: Id[];
+  id?: Id;
   payload: T;
 }, ack: MessageAck, nack: MessageNack) => Promise<void>;
 interface AdditionalIdsHookData<T = any> {
@@ -246,8 +247,10 @@ interface BulkRouteOptions<T = any> {
   /** Sequelize model from which to scan IDs */
   model: Model<any, any> | ModelStatic<any> | any;
   modelScopes: string[];
-  /** Per-ID handler that performs the operation */
+  /** Consumer that processes IDs (can handle single ID [id] or multiple [id1, id2, ...]) */
   consumer: BulkPerIdHandler<T>;
+  /** Consumer batch size - how many IDs to send to consumer at once (default 1) */
+  consumerBatchSize?: number;
   consumerOptions?: ConsumeOptions;
   rabbitQueueName?: string;
   payloadSchema?: z$1.ZodType<T>;
@@ -362,6 +365,7 @@ declare class BulkRoute<T = any> {
   private readonly consumer;
   private readonly idField;
   private readonly pageSize;
+  private readonly consumerBatchSize;
   private readonly rabbitQueueName;
   private readonly rabbit;
   private readonly consumerOptions;
@@ -372,6 +376,8 @@ declare class BulkRoute<T = any> {
   bulkHandler: Asyncify<Handler>;
   /** Get user jobs - public method for BulkRouter to use */
   getUserJobs(userId: string, limit?: number): Promise<(JobStatus | null)[]>;
+  /** Helper method to batch IDs into chunks of consumerBatchSize */
+  private batchIds;
   private rabbitScanAndEnqueue;
   startRabbitWorker(): Promise<void>;
 }
@@ -530,12 +536,12 @@ declare class JobManager {
   * Handle successful message processing (ack)
   * Uses Redis Lua script for atomic operation
   */
-  ack(jobId: string): Promise<void>;
+  ack(jobId: string, count?: number): Promise<void>;
   /**
   * Handle failed message processing (nack)
   * Uses Redis Lua script for atomic operation
   */
-  nack(jobId: string, errorMsg: string, data: any): Promise<void>;
+  nack(jobId: string, errorMsg: string, data: any, count?: number): Promise<void>;
   /**
   * Add a job to user's job list
   */
@@ -577,6 +583,7 @@ interface BulkerInit {
     pageSize?: number;
     maxJobsPerUser?: number;
     idField?: string;
+    consumerBatchSize?: number;
     workerConcurrency?: number;
     jobTtlSeconds?: number;
     errorLogLimit?: number;

package/lib/index.js

@@ -1,13 +1,14 @@
-import{Op as e,literal as t}from"sequelize";import n from"@autofleet/logger";import{BadRequest as r,UnexpectedError as i,handleError as a}from"@autofleet/errors";import o from"joi";import{customFields as s}from"@autofleet/common-types";import{randomInt as c,randomUUID as l}from"node:crypto";import{createClient as u}from"redis";import{Router as d}from"express";import{z as f,z as p}from"zod";import{EventEmitter as m}from"node:events";const h=[`eq`,`ne`,`gte`,`gt`,`lte`,`lt`,`not`,`in`,`notIn`,`is`,`like`,`iLike`,`notLike`,`between`,`and`,`or`,`overlap`,`contains`],g={$eq:`=`,$ne:`!=`,$gte:`>=`,$gt:`>`,$lte:`<=`,$lt:`<`,$not:`NOT`,$in:`IN`,$notIn:`NOT IN`,$is:`IS`,$like:`LIKE`,$iLike:`ILIKE`,$notLike:`NOT LIKE`,$and:`AND`,$or:`OR`},_=(t={Op:e})=>{let{Op:n}=t;return Object.fromEntries(h.map(e=>[`${`$`+e}`,n[e]]))},v=e=>`\$${e}\$`,y=(e,t)=>e.includes(`.`)&&t.includes(e.split(`.`,1)[0]),b=(e,t)=>{let n=e;return e.includes(`-`)&&([,n]=n.split(`-`,2)),y(n,t)&&([n]=n.split(`.`,1)),n},x=e=>e.includes(`-`),S=e=>{throw new r([Error(e)])},C=e=>e.split(`.`,2)[1],w=(e=5)=>Array.from({length:e},()=>`ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz`.charAt(c(52))).join(``),T=(e,t)=>Object.fromEntries(t.map(t=>[t,e[t]])),E={length:`length`};function D(e){return e.replace(/(?!^)[A-Z]/g,e=>`_${e.toLowerCase()}`)}function ee(e){switch(e.action){case E.length:return[t(`jsonb_array_length(${D(e.columnName)})`),e.alias];default:return e.action,[]}}function te(e){return e.map(e=>ee(e))}function ne(e){return e.map(e=>{let n=D(e.columnName),r=`json_build_object(${e.keys.map(e=>`'${e}', ${n} -> '${e}'`).join(`, `)})`,i=e.alias||e.columnName;return[t(r),i]})}function O({select:e=[],computed:t=[]}={}){let n=ne(e),r=te(t);return[...n,...r]}const k=`DESC`,A=`customFields.`,{CUSTOM_FIELDS_FILTER_SCOPE:j,CUSTOM_FIELDS_SORT_SCOPE:M}=s,N=e=>[`string`,`number`].includes(typeof e)||Array.isArray(e)?e:Object.entries(e).map(([e,t])=>({operator:g[e],value:t})),re=(e,t={})=>{let{literalAttributes:n=[],DBFormatter:r=void 0}=t,[i,a]=e.reduce((e,t)=>{let[i,a=`ASC`]=Array.isArray(t)?t:[t],o=n?.find(e=>e.attribute===i);if(o){let t=r?r(`"${o.attribute}" ${a}`):`${o.attribute} ${a}`;e[1].push(o.literal),e[0].push([t])}else e[0].push(t);return e},[[],[]]);return[i,a]},P=e=>{let t={};return Object.entries(e).forEach(([e,n])=>{let r=w();if(t[r]=e.split(A,2)[1],Array.isArray(n))n.forEach(e=>{let n=w();t[n]=typeof e==`string`?e:e.value});else if(typeof n==`string`||typeof n==`number`){let e=w();t[e]=n}else if(n?.operator){let e=w();t[e]=n.value}}),t},ie=e=>{let t={};return e.forEach(e=>{if(e.startsWith(A)){let n=w();t[n]=e.split(A,2)[1]}else if(e.substring(1).startsWith(A)){let n=w();t[n]=e.substring(1).split(A,2)[1]}}),t},ae=(e,t)=>({...ie(e),...P(t)}),oe=({order:e,associationModels:t=[],replacementsMap:n={}})=>{let r=[],i=new Map;return e.forEach(e=>{if([e,e.substring(1)].some(e=>e.startsWith(A))){i.has(M)||i.set(M,{});let t=e.split(A,2)[1];i.get(M)[t]=x(e)?k:`ASC`;return}let n=[b(e,t)],a=x(e);y(a?e.split(`-`,2)[1]:e,t)&&n.push(C(e)),a&&n.push(k),r.push(n)}),{formattedOrders:r,replacementsMap:n,orderScopes:Array.from(i.entries()).map(([e,t])=>t?{method:[e,{replacementsMap:n,scopeValue:t}]}:e)}},se=e=>e||1,ce=e=>e||20,F=(e,t={})=>{let n=e.map(e=>{let n=t[typeof e==`string`?e:e.association||e.model];return{...typeof e!=`string`&&e,association:n,required:typeof e==`string`||e.required!==!1,...typeof e!=`string`&&e.include&&{include:F(e.include,n?.target?.associations)}}});return n=n.map(({model:e,...t})=>t),n},le=(e,t,n,r=[])=>{let i={},a={},o=new Map;return Object.entries(e).forEach(([e,n])=>{if(e.startsWith(A)){o.has(j)||o.set(j,{});let t=e.split(A,2)[1];o.get(j)[t]=N(n);return}if(r.includes(e)){a[e]=n;return}let s=y(e,t)?v(e):e;i[s]=n}),{formattedQuery:i,externalQueryValues:a,formattedScopes:Array.from(o.entries()).map(([e,t])=>t?{method:[e,{replacementsMap:n,scopeValue:t}]}:e)}},ue=(e,t,n)=>({$and:e.split(` `).map(e=>({$or:t.filter(e=>n[e].type.key===`STRING`).map(t=>({[t]:{$iLike:`%${e}%`}}))}))});var de=({order:e=[],page:t=1,perPage:n=20,include:r=[],query:i={},attributes:a=null,searchTerm:o=null,jsonAttributes:s={}},c,l)=>{let u=ae(e,i),d=Object.keys(c?.associations||{}),{formattedOrders:f,orderScopes:p}=oe({order:[...e,`id`],associationModels:d,replacementsMap:u}),[m,h]=re(f,l),g=O(s),_=[...h,...a??[],...g],v=a?.length?_:{include:_},y=F(r,c?.associations),b=se(t),x=ce(n),S=le(i,d,u,l?.additionalAllowedAttributes),{formattedScopes:C,externalQueryValues:w}=S,{formattedQuery:T}=S;if(o&&!l?.skipSearchTermFormat){let e=ue(o,a?.length?a:Object.keys(c.rawAttributes||{}),c.rawAttributes);T=!T||Object.keys(T).length===0?e:{$and:[T,e]}}return{query:T,order:m,page:b,perPage:x,include:y,scopes:[...C,...p],...v&&{attributes:v},...Object.keys(w).length>0&&{externalQueryValues:w}}};const fe=e=>h.includes(e.split(`$`,2)[1]),I=(e,t=[],n=[],r=[])=>{let i=e.startsWith(`$`)&&e.endsWith(`$`)?e.slice(1,-1):e;return[...t,...n].includes(i.includes(`.`)?i.split(`.`,1)[0]:i)||r.includes(i)},L=(e,t,n,r={})=>{let i=x(e);i&&!e.startsWith(`-`)&&S(`- must be only at the beginning of the word`);let a=i?e.split(`-`,2)[1]:e,o=y(a,n),s=b(e,n),c=r?.literalAttributes?.map(e=>e.attribute)?.includes(a);!o&&s.includes(`.`)&&([s]=s.split(`.`,1)),t.includes(s)||o||c||S(`${e} is invalid. isLiteralAttribute: ${c}`)},R=(e,t)=>{t.includes(e)||S(`${e} is invalid`)},z=(e,t,n=[],r={})=>{e.forEach(e=>L(e,t,n,r))},B=(e,t)=>{e.forEach(e=>R(e,t))},V=(e,t)=>{B([...e.select?.map(e=>e.columnName)??[],...e.computed?.map(e=>e.columnName)??[]],t)},pe=(e,t)=>{let n=Array.isArray(e)?e:Object.keys(e);if(!n?.length)return;let r=n.find(e=>!t?.enrichmentAttributes?.includes(e));r&&S(`enrichment attribute ${r} is invalid`)},H=(e,t,n=[],r=[])=>{Object.entries(e).forEach(([e,i])=>{Array.isArray(i)?i[0]&&typeof i[0]==`object`&&i.map(e=>H(e,t,n,r)):fe(e)||I(e,t,n,r)?i&&typeof i==`object`&&H(i,t,[],r):S(`invalid key: ${e}`)})},me=({page:e,perPage:t})=>{e<1&&S(`Page must be greater than 0`),(t>100||t<1)&&S(`PerPage must be between 1 to 100`)},he=(e,t)=>{let n=Object.keys(t);e.forEach(e=>{I(e.model,n);let r=t[e.model]?.target;r||S(`model not found in associations`);let{rawAttributes:i}=r,a=Object.keys(i);e.where&&H(e.where,a),e.order&&z(e.order,a),e.attributes&&B(e.attributes,a),[null,void 0,!0,!1].includes(e.required)||S(`include.required must be a boolean`)})},U=({query:e={},order:t=[],attributes:n=[],include:r=[],page:i=1,perPage:a=20,enrichments:o=[],group:s=[],jsonAttributes:c={}},l,u={})=>{let d=Object.keys(l.rawAttributes),f=Object.keys(l?.associations||{});return!n||n.length===0?n=d:B(n,d),z(t,d,f,u),H(e,d,f,u.additionalAllowedAttributes),pe(o,u),V(c,d),Array.isArray(s)||S(`group must be an array`),r.length&&typeof r==`object`?he(r,l?.associations):r&&typeof r!=`object`&&S(`include must be an array`),me({page:i,perPage:a}),!0},{object:W,string:G,number:K,any:ge,array:q,alternatives:_e}=o.types(),ve=n(),ye=W.keys({query:W,attributes:q.items(G),order:q.items(G),page:K,perPage:K,include:q.items(ge),searchTerm:G,group:q.items(G),enrichments:_e.try(q.items(G),W.pattern(G,{exclude:q.items(G)})),jsonAttributes:o.object({select:o.array().items(o.object({columnName:o.string().required(),keys:o.array().items(o.string().required()).required(),alias:o.string().optional()})).default([]),computed:o.array().items(o.object({columnName:o.string().required(),action:o.string().valid(...Object.values(E)).required(),alias:o.string().required()})).default([])}).default({})}),J=(e,t,n={})=>{let{query:i,attributes:a,order:o,page:s,perPage:c,include:l,group:u,enrichments:d,jsonAttributes:f}=t,p=ye.validate(t);if(p.error)throw new r([p.error]);U({query:i,attributes:a,order:o,page:s,perPage:c,include:l,enrichments:d,group:u,jsonAttributes:f},e,n)},be=(e,t={},n=`body`)=>(r,i,o)=>{try{J(e,r[n],t),o()}catch(e){let{query:o,attributes:s,order:c}=r[n];a(e,i,{logger:t.logger??ve,message:`error in query middleware`,payload:{error:e,query:o,attributes:s,order:c}})}},Y=(e,t,n={})=>{let{order:r,page:i,perPage:a,include:o,query:s,attributes:c,searchTerm:l,jsonAttributes:u}=t,{query:d,externalQueryValues:f,order:p,page:m,perPage:h,include:g,scopes:_,attributes:v}=de({query:s,order:r,page:i,perPage:a,include:o,attributes:c,searchTerm:l,jsonAttributes:u},e,n);t.query=d,t.externalQueryValues=f,t.order=p,t.attributes=v,t.page=m,t.perPage=h,t.include=g,t.scopes=_,n.includeRawPayload&&(t.rawPayload={order:r,page:i,perPage:a,include:o,query:s,attributes:c,searchTerm:l})},xe=(e,t={},n=`body`)=>(r,i,a)=>{Y(e,r[n],t),a()},Se=({model:e,logger:t,validationOptions:n,formatOptions:r,modelName:o=e.constructor?.name,additionalScopes:s=[],modifyQueryValues:c,onRowsRetrieved:l})=>async(u,d)=>{try{J(e,u.body,{...n,logger:t})}catch(e){a(e,d,{logger:t,message:`error in query endpoint`,payload:T(u.body,[`query`,`order`,`attributes`])});return}try{Y(e,u.body,r);let n=Object.assign(T(u.body,[`query`,`externalQueryValues`,`order`,`attributes`,`page`,`perPage`,`include`,`scopes`,`enrichments`]),{distinct:!0});t.info(`querying ${o}`,{queryValues:n});let i=c?.(n)??n,{scopes:a=[],query:f,perPage:p,page:m,enrichments:h,externalQueryValues:g,..._}=i,v=await e.scope([...s,...a]).findAndCountAll({where:f,limit:p,offset:(m-1)*p,..._});if(!v.rows.length||!l){d.json(v);return}let y=await l(v,i);d.json(y)}catch(e){a(new i(e),d,{logger:t,message:`Error while querying ${o}`,payload:{query:u.body}})}};var X=class e extends Error{constructor(e,t=!1){super(e),this.name=`BulkerError`,this.retryable=t,Object.setPrototypeOf(this,new.target.prototype)}static isBulkerError(t){return t instanceof e}static wrap(t,n,r=!1){return t instanceof e?new e(n??t.message,r||t.retryable):new e(n?`${n}: ${t instanceof Error?t.message:String(t)}`:String(t),r)}static retryable(t){return new e(t,!0)}static nonRetryable(t){return new e(t,!1)}};const Ce=e=>e.length===0?`No keys provided`:e.length===1?`Key "${e[0]}" is required`:`Exactly one of [${e.join(`, `)}] must be provided`;function we(e,t){return p.object(e).refine(e=>t.filter(t=>e[t]!==void 0&&e[t]!==null).length===1,{message:Ce(t)})}const Z=p.string().uuid();function Q(e){return we(Object.fromEntries(e.map(e=>[e,p.union([Z,p.array(Z)]).optional()])),e)}const Te=[`businessModelId`,`fleetId`,`demandSourceId`,`contextId`,`userId`,`businessAccountId`,`activeBusinessModelId`];var Ee=class{constructor(e,t){if(this.bulker=e,this.opts=t,this.rabbitQueueName=null,this.rabbit=null,this.bulkHandler=async(e,t,n)=>{try{let{query:n,payload:r,preview:i}=e.body??{},a=r;if(r&&this.payloadSchema)try{a=this.payloadSchema.parse(r)}catch(e){if(e instanceof p.ZodError)return t.status(400).json({error:`invalid_payload`,details:e.issues.map(e=>`${e.path.join(`.`)}: ${e.message}`).join(`, `)});throw e}try{J(this.model,n,{logger:this.bulker.logger})}catch(e){return t.status(400).json({error:`invalid_query`,details:e.message||e})}let o=this.identityScopeSchema.safeParse(n?.query||{});if(!o.success)return t.status(400).json({error:`invalid_identity_scope`,details:o.error.issues.map(e=>`${e.path.join(`.`)}: ${e.message}`).join(`, `)});Y(this.model,n,{includeRawPayload:!0});let{query:s,...c}=Object.assign(T(n,[`query`,`externalQueryValues`,`order`,`include`,`scopes`,`enrichments`]),{distinct:!0}),u=[];if(this.opts.additionalIdsHook){let{rawPayload:e}=n;try{u=await this.opts.additionalIdsHook({rawPayload:e,payload:a}),this.bulker.logger.info(`additionalIdsHook returned ${u.length} IDs for action ${this.action}`)}catch(e){return this.bulker.logger.error(`Error in additionalIdsHook for action ${this.action}: ${e.message||e}`,{err:e}),t.status(500).json({error:`additional_ids_hook_error`,details:e.message||e})}}let d=[];s&&typeof s==`object`&&Object.keys(s).length>0&&d.push(s),Array.isArray(u)&&u.length>0&&d.push({id:{$in:u}});let f;if(d.length===0)return t.status(400).json({error:`no_query`,details:`No valid query provided to select records`});if(d.length===1){let[e]=d;f=e}else f={$or:d};this.bulker.logger.info(`Constructed final where clause for action ${this.action}`,{where:f});let m={where:f,...c},h=await this.model.scope(this.modelScopes).count({...m,col:this.idField});if(i)return t.json({estimatedCount:h});let g=l();return await this.bulker.jobManager.initJob(g,{status:`queued`,total:h,action:this.action}),this.bulker.emitEvent(`job:created`,{jobId:g,action:this.action,total:h}),setImmediate(()=>{this.rabbitScanAndEnqueue(g,m,a).catch(e=>this.bulker.logger.error(e))}),t.status(202).json({jobId:g,estimatedCount:h})}catch(e){return this.bulker.logger.error(`Error in bulkHandler for action ${this.action}: ${e.message||e}`,{err:e}),n(e)}},this.action=t.action,this.model=t.model,this.modelScopes=t.modelScopes??[],this.consumer=t.consumer,this.rabbit=e.rabbit,this.rabbitQueueName=t.rabbitQueueName??`bulk-${this.action}-queue`,this.consumerOptions={enableRabbitTrace:!0,...this.opts.consumerOptions},this.payloadSchema=t.payloadSchema,!/^[a-zA-Z0-9-_]+$/.test(this.action))throw Error(`BulkRoute action must be alphanumeric`);if(!this.model||typeof this.model.findAll!=`function`||typeof this.model.count!=`function`)throw Error(`BulkRoute model must be a valid Sequelize model`);if(typeof this.consumer!=`function`)throw Error(`BulkRoute consumer must be a function`);if(this.payloadSchema&&!(this.payloadSchema instanceof p.ZodType))throw Error(`BulkRoute payloadSchema must be a Zod schema`);if(this.modelScopes&&!Array.isArray(this.modelScopes))throw Error(`BulkRoute scopes must be an array of strings`);if(this.opts.additionalIdsHook&&typeof this.opts.additionalIdsHook!=`function`)throw Error(`BulkRoute additionalIdsHook must be a function`);this.idField=t.idField??e.defaults.idField,this.pageSize=t.pageSize??e.defaults.pageSize;let n=this.model.rawAttributes||{},r=(t.identityScopes&&t.identityScopes.length>0?t.identityScopes:Te).filter(e=>!!n[e]);r.length===0&&this.bulker.logger.warn(`BulkRoute for action ${this.action} has no valid identityScopes configured - all records will be accessible`),this.bulker.logger.info(`BulkRoute for action ${this.action} using idField ${this.idField}, pageSize ${this.pageSize}, identityScopes: ${r.join(`, `)}`),this.identityScopeSchema=Q(r),this.startRabbitWorker().catch(e=>{this.bulker.logger.error(`Failed to start RabbitMQ worker for queue ${this.rabbitQueueName}: ${e.message||e}`)})}async getUserJobs(e,t=20){return this.bulker.jobManager.getUserJobs(e,t)}async rabbitScanAndEnqueue(t,n,r){if(!this.rabbit)throw Error(`RabbitMQ not configured in Bulker`);let i=Date.now().toString();if(this.bulker.getUserId){let e=this.bulker.getUserId();e&&await this.bulker.jobManager.addUserJob(e,t)}await this.bulker.jobManager.setJobFields(t,{status:`running`,startTime:i}),this.bulker.emitEvent(`job:started`,{jobId:t,action:this.action});let a=null,o=0,s=0;for(;;){let i=await this.bulker.jobManager.getJobField(t,`status`);if(!i||i===`canceled`){this.bulker.logger.info(`Job ${t} was canceled, stopping scan and enqueue`);break}let c=a?{[e.and]:[n.where,{[this.idField]:{[e.gt]:a}}]}:n.where,l=await this.model.scope(this.modelScopes).findAll({where:c,...T(n,[`include`,`order`,`scopes`]),attributes:[this.idField],order:[[this.idField,`ASC`]],limit:this.pageSize,raw:!0,subQuery:!1});if(l.length===0)break;let u=l.map(e=>({jobId:t,id:e[this.idField],payload:r}));this.bulker.logger.info(`Enqueuing ${u.length} messages to RabbitMQ queue ${this.rabbitQueueName}`);let d=(await Promise.allSettled(u.map(e=>this.rabbit.sendToQueue(this.rabbitQueueName,e)))).filter(e=>e.status===`rejected`);if(d.length>0)throw this.bulker.logger.error(`Failed to enqueue ${d.length} messages to RabbitMQ`,{rejected:d}),Error(`Failed to enqueue ${d.length} messages to RabbitMQ`);o+=l.length,await this.bulker.jobManager.incrJobField(t,`queued`,l.length),a=l[l.length-1][this.idField],s+=1,this.bulker.emitEvent(`scan:page`,{jobId:t,action:this.action,pageNumber:s,itemsInPage:l.length});let f=await this.bulker.jobManager.getJob(t);f&&this.bulker.emitEvent(`job:queued`,{jobId:t,action:this.action,queued:o,total:f.total})}o===0&&await this.bulker.jobManager.completeEmptyJob(t)}async startRabbitWorker(){return this.bulker.logger.info(`Starting RabbitMQ consumer for queue ${this.rabbitQueueName}`),this.bulker.emitEvent(`worker:started`,{action:this.action,queueName:this.rabbitQueueName}),this.rabbit?.consume(this.rabbitQueueName,async(e,t,n)=>{if(!e)return;let{jobId:r,id:i,payload:a}=e.content,o=!1,s=async()=>{o||(o=!0,await Promise.all([this.bulker.jobManager.ack(r),t()]),this.bulker.emitEvent(`item:processed`,{jobId:r,action:this.action,id:i}))},c=async(e,t)=>{if(o)return;o=!0;let s=t;s===void 0&&(s=X.isBulkerError(e)?e.retryable:!1),await Promise.all([this.bulker.jobManager.nack(r,e?e.message||String(e):`nacked`,{id:i,payload:a}),n(void 0,{skipRetry:!s})]),s?this.bulker.emitEvent(`item:retrying`,{jobId:r,action:this.action,id:i}):this.bulker.emitEvent(`item:failed`,{jobId:r,action:this.action,id:i,error:e?e.message||String(e):`nacked`})},l=await this.bulker.jobManager.getJobField(r,`status`);if(!l||l===`canceled`){await this.bulker.jobManager.incrJobField(r,`processed`,1);return}try{this.bulker.logger.info(`Processing job ${r} action ${this.action} id ${i}`),this.bulker.emitEvent(`item:processing`,{jobId:r,action:this.action,id:i}),await this.consumer({jobId:r,id:i,payload:a},s,c),await s()}catch(e){this.bulker.logger.error(`Error processing job ${r} action ${this.action} id ${i}: ${e.message||e}`,{err:e}),this.bulker.emitEvent(`worker:error`,{action:this.action,error:e.message||String(e)}),await c(e,!1)}},this.consumerOptions)}},De=class{constructor(e,t,n=`/bulk-actions`){this.routes=[],this.actionsToRouteMap=new Map,this.bulkHandler=async(e,t,n)=>{try{let{action:r}=e.body??{};if(!r)return t.status(400).json({error:`missing_action`,message:`Request body must include an "action" field`});let i=this.actionsToRouteMap.get(r);return i?i.bulkHandler(e,t,n):t.status(404).json({error:`unknown_action`,message:`Action "${r}" is not registered`,availableActions:Array.from(this.actionsToRouteMap.keys())})}catch(e){return this.bulker.logger.error(`Error in BulkRouter bulkHandler: ${e.message||e}`,{err:e}),n(e),null}},this.getJobHandler=async(e,t,n)=>{try{if(!e.params.id)return t.status(400).json({error:`missing_id`});let n=await this.bulker.jobManager.getJob(e.params.id);return n?t.json(n):t.status(404).json({error:`not_found`})}catch(e){return n(e)}},this.cancelJobHandler=async(e,t,n)=>{try{return e.params.id?await this.bulker.jobManager.cancelJob(e.params.id)?(this.bulker.logger.info(`Job ${e.params.id} cancel requested`),t.json({ok:!0})):t.status(404).json({error:`not_found`}):t.status(400).json({error:`missing_id`})}catch(e){return n(e)}},this.getMyJobsHandler=async(e,t,n)=>{try{if(!this.bulker.getUserId)return t.status(400).json({error:`user_id_function_not_configured`,message:`Bulker instance does not have a getUserId function configured`});let e=this.bulker.getUserId();if(!e)return t.json([]);let n=(await this.bulker.jobManager.getUserJobs(e)).filter(e=>e!==null);return n.sort((e,t)=>{let n=e.createdAt?new Date(e.createdAt).getTime():0;return(t.createdAt?new Date(t.createdAt).getTime():0)-n}),t.json(n)}catch(e){return n(e)}},this.bulker=e,this.router=t??d(),this.staticRoute=n,this.registerStaticRoutes()}registerStaticRoutes(){this.router.post(this.staticRoute,this.bulkHandler),this.router.get(`/jobs/:id`,this.getJobHandler),this.router.get(`/jobs`,this.getMyJobsHandler),this.router.post(`/jobs/:id/cancel`,this.cancelJobHandler)}addAction(e,t){let n=new Ee(this.bulker,{action:e,...t});if(this.bulker.logger.info(`Registering action handler: ${e}`),this.actionsToRouteMap.has(e))throw Error(`Action "${e}" is already registered`);return this.actionsToRouteMap.set(e,n),this.routes.push(n),n}getRouter(){return this.router}},$=class e{constructor(e,t){this.redis=e,this.defaults={maxJobsPerUser:t?.maxJobsPerUser??-1,jobTtlSeconds:t?.jobTtlSeconds??168*3600,errorLogLimit:t?.errorLogLimit??10}}setEventEmitter(e){this.eventEmitter=e}static jobKey(e){return`bulker:job:${e}`}static userJobsKey(e){return`bulker:user:${e}:jobs`}async initJob(t,n){let r=Date.now().toString(),i=e.jobKey(t),a=this.redis.multi();a.hSet(i,{status:n.status,total:String(n.total),queued:`0`,processed:`0`,succeeded:`0`,failed:`0`,action:n.action,errors:JSON.stringify([]),createdAt:r,updatedAt:r,startTime:r,endTime:``}),a.expire(i,this.defaults.jobTtlSeconds),await a.exec()}async getJob(t){let n=e.jobKey(t),r=await this.redis.hGetAll(n);if(!r||Object.keys(r).length===0)return null;let i=r.startTime?Number(r.startTime):null,a=r.endTime&&r.endTime!==``?Number(r.endTime):null,o=Date.now(),s={startTime:i?new Date(i).toISOString():null,endTime:a?new Date(a).toISOString():null,durationMs:i?(a||o)-i:null};return{jobId:t,status:r.status,action:r.action,total:Number(r.total??0),queued:Number(r.queued??0),processed:Number(r.processed??0),succeeded:Number(r.succeeded??0),failed:Number(r.failed??0),errors:r.errors?JSON.parse(r.errors):[],createdAt:r.createdAt?new Date(Number(r.createdAt)).toISOString():void 0,updatedAt:r.updatedAt?new Date(Number(r.updatedAt)).toISOString():void 0,duration:s}}async setJobField(t,n,r){let i=e.jobKey(t);if(!await this.redis.exists(i))return!1;let a=this.redis.multi();return a.hSet(i,n,r),a.hSet(i,`updatedAt`,Date.now().toString()),await a.exec(),!0}async setJobFields(t,n){let r=e.jobKey(t);if(!await this.redis.exists(r))return!1;let i=this.redis.multi();return i.hSet(r,n),i.hSet(r,`updatedAt`,Date.now().toString()),await i.exec(),!0}async incrJobField(t,n,r=1){let i=e.jobKey(t),a=this.redis.multi();a.hIncrBy(i,n,r),a.hSet(i,`updatedAt`,Date.now().toString());let o=await a.exec();return o?.[0]&&o[0][0]===null?o[0][1]:0}async getJobField(t,n){let r=e.jobKey(t);return this.redis.hGet(r,n)}async cancelJob(e){return this.setJobField(e,`status`,`canceled`)}async completeEmptyJob(e){let t=Date.now().toString();await this.setJobFields(e,{status:`completed`,endTime:t})}async ack(t){let n=e.jobKey(t);await this.redis.eval(`
+import{Op as e,literal as t}from"sequelize";import n from"@autofleet/logger";import{BadRequest as r,UnexpectedError as i,handleError as a}from"@autofleet/errors";import o from"joi";import{customFields as s}from"@autofleet/common-types";import{randomInt as c,randomUUID as l}from"node:crypto";import{createClient as u}from"redis";import{Router as d}from"express";import{z as f,z as p}from"zod";import{EventEmitter as m}from"node:events";const h=[`eq`,`ne`,`gte`,`gt`,`lte`,`lt`,`not`,`in`,`notIn`,`is`,`like`,`iLike`,`notLike`,`between`,`and`,`or`,`overlap`,`contains`],g={$eq:`=`,$ne:`!=`,$gte:`>=`,$gt:`>`,$lte:`<=`,$lt:`<`,$not:`NOT`,$in:`IN`,$notIn:`NOT IN`,$is:`IS`,$like:`LIKE`,$iLike:`ILIKE`,$notLike:`NOT LIKE`,$and:`AND`,$or:`OR`},_=(t={Op:e})=>{let{Op:n}=t;return Object.fromEntries(h.map(e=>[`${`$`+e}`,n[e]]))},v=e=>`\$${e}\$`,y=(e,t)=>e.includes(`.`)&&t.includes(e.split(`.`,1)[0]),b=(e,t)=>{let n=e;return e.includes(`-`)&&([,n]=n.split(`-`,2)),y(n,t)&&([n]=n.split(`.`,1)),n},x=e=>e.includes(`-`),S=e=>{throw new r([Error(e)])},C=e=>e.split(`.`,2)[1],w=(e=5)=>Array.from({length:e},()=>`ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz`.charAt(c(52))).join(``),T=(e,t)=>Object.fromEntries(t.map(t=>[t,e[t]])),E={length:`length`};function D(e){return e.replace(/(?!^)[A-Z]/g,e=>`_${e.toLowerCase()}`)}function ee(e){switch(e.action){case E.length:return[t(`jsonb_array_length(${D(e.columnName)})`),e.alias];default:return e.action,[]}}function te(e){return e.map(e=>ee(e))}function ne(e){return e.map(e=>{let n=D(e.columnName),r=`json_build_object(${e.keys.map(e=>`'${e}', ${n} -> '${e}'`).join(`, `)})`,i=e.alias||e.columnName;return[t(r),i]})}function O({select:e=[],computed:t=[]}={}){let n=ne(e),r=te(t);return[...n,...r]}const k=`DESC`,A=`customFields.`,{CUSTOM_FIELDS_FILTER_SCOPE:j,CUSTOM_FIELDS_SORT_SCOPE:M}=s,N=e=>[`string`,`number`].includes(typeof e)||Array.isArray(e)?e:Object.entries(e).map(([e,t])=>({operator:g[e],value:t})),re=(e,t={})=>{let{literalAttributes:n=[],DBFormatter:r=void 0}=t,[i,a]=e.reduce((e,t)=>{let[i,a=`ASC`]=Array.isArray(t)?t:[t],o=n?.find(e=>e.attribute===i);if(o){let t=r?r(`"${o.attribute}" ${a}`):`${o.attribute} ${a}`;e[1].push(o.literal),e[0].push([t])}else e[0].push(t);return e},[[],[]]);return[i,a]},P=e=>{let t={};return Object.entries(e).forEach(([e,n])=>{let r=w();if(t[r]=e.split(A,2)[1],Array.isArray(n))n.forEach(e=>{let n=w();t[n]=typeof e==`string`?e:e.value});else if(typeof n==`string`||typeof n==`number`){let e=w();t[e]=n}else if(n?.operator){let e=w();t[e]=n.value}}),t},ie=e=>{let t={};return e.forEach(e=>{if(e.startsWith(A)){let n=w();t[n]=e.split(A,2)[1]}else if(e.substring(1).startsWith(A)){let n=w();t[n]=e.substring(1).split(A,2)[1]}}),t},ae=(e,t)=>({...ie(e),...P(t)}),oe=({order:e,associationModels:t=[],replacementsMap:n={}})=>{let r=[],i=new Map;return e.forEach(e=>{if([e,e.substring(1)].some(e=>e.startsWith(A))){i.has(M)||i.set(M,{});let t=e.split(A,2)[1];i.get(M)[t]=x(e)?k:`ASC`;return}let n=[b(e,t)],a=x(e);y(a?e.split(`-`,2)[1]:e,t)&&n.push(C(e)),a&&n.push(k),r.push(n)}),{formattedOrders:r,replacementsMap:n,orderScopes:Array.from(i.entries()).map(([e,t])=>t?{method:[e,{replacementsMap:n,scopeValue:t}]}:e)}},se=e=>e||1,ce=e=>e||20,F=(e,t={})=>{let n=e.map(e=>{let n=t[typeof e==`string`?e:e.association||e.model];return{...typeof e!=`string`&&e,association:n,required:typeof e==`string`||e.required!==!1,...typeof e!=`string`&&e.include&&{include:F(e.include,n?.target?.associations)}}});return n=n.map(({model:e,...t})=>t),n},le=(e,t,n,r=[])=>{let i={},a={},o=new Map;return Object.entries(e).forEach(([e,n])=>{if(e.startsWith(A)){o.has(j)||o.set(j,{});let t=e.split(A,2)[1];o.get(j)[t]=N(n);return}if(r.includes(e)){a[e]=n;return}let s=y(e,t)?v(e):e;i[s]=n}),{formattedQuery:i,externalQueryValues:a,formattedScopes:Array.from(o.entries()).map(([e,t])=>t?{method:[e,{replacementsMap:n,scopeValue:t}]}:e)}},ue=(e,t,n)=>({$and:e.split(` `).map(e=>({$or:t.filter(e=>n[e].type.key===`STRING`).map(t=>({[t]:{$iLike:`%${e}%`}}))}))});var de=({order:e=[],page:t=1,perPage:n=20,include:r=[],query:i={},attributes:a=null,searchTerm:o=null,jsonAttributes:s={}},c,l)=>{let u=ae(e,i),d=Object.keys(c?.associations||{}),{formattedOrders:f,orderScopes:p}=oe({order:[...e,`id`],associationModels:d,replacementsMap:u}),[m,h]=re(f,l),g=O(s),_=[...h,...a??[],...g],v=a?.length?_:{include:_},y=F(r,c?.associations),b=se(t),x=ce(n),S=le(i,d,u,l?.additionalAllowedAttributes),{formattedScopes:C,externalQueryValues:w}=S,{formattedQuery:T}=S;if(o&&!l?.skipSearchTermFormat){let e=ue(o,a?.length?a:Object.keys(c.rawAttributes||{}),c.rawAttributes);T=!T||Object.keys(T).length===0?e:{$and:[T,e]}}return{query:T,order:m,page:b,perPage:x,include:y,scopes:[...C,...p],...v&&{attributes:v},...Object.keys(w).length>0&&{externalQueryValues:w}}};const fe=e=>h.includes(e.split(`$`,2)[1]),I=(e,t=[],n=[],r=[])=>{let i=e.startsWith(`$`)&&e.endsWith(`$`)?e.slice(1,-1):e;return[...t,...n].includes(i.includes(`.`)?i.split(`.`,1)[0]:i)||r.includes(i)},L=(e,t,n,r={})=>{let i=x(e);i&&!e.startsWith(`-`)&&S(`- must be only at the beginning of the word`);let a=i?e.split(`-`,2)[1]:e,o=y(a,n),s=b(e,n),c=r?.literalAttributes?.map(e=>e.attribute)?.includes(a);!o&&s.includes(`.`)&&([s]=s.split(`.`,1)),t.includes(s)||o||c||S(`${e} is invalid. isLiteralAttribute: ${c}`)},R=(e,t)=>{t.includes(e)||S(`${e} is invalid`)},z=(e,t,n=[],r={})=>{e.forEach(e=>L(e,t,n,r))},B=(e,t)=>{e.forEach(e=>R(e,t))},V=(e,t)=>{B([...e.select?.map(e=>e.columnName)??[],...e.computed?.map(e=>e.columnName)??[]],t)},pe=(e,t)=>{let n=Array.isArray(e)?e:Object.keys(e);if(!n?.length)return;let r=n.find(e=>!t?.enrichmentAttributes?.includes(e));r&&S(`enrichment attribute ${r} is invalid`)},H=(e,t,n=[],r=[])=>{Object.entries(e).forEach(([e,i])=>{Array.isArray(i)?i[0]&&typeof i[0]==`object`&&i.map(e=>H(e,t,n,r)):fe(e)||I(e,t,n,r)?i&&typeof i==`object`&&H(i,t,[],r):S(`invalid key: ${e}`)})},me=({page:e,perPage:t})=>{e<1&&S(`Page must be greater than 0`),(t>100||t<1)&&S(`PerPage must be between 1 to 100`)},he=(e,t)=>{let n=Object.keys(t);e.forEach(e=>{I(e.model,n);let r=t[e.model]?.target;r||S(`model not found in associations`);let{rawAttributes:i}=r,a=Object.keys(i);e.where&&H(e.where,a),e.order&&z(e.order,a),e.attributes&&B(e.attributes,a),[null,void 0,!0,!1].includes(e.required)||S(`include.required must be a boolean`)})},U=({query:e={},order:t=[],attributes:n=[],include:r=[],page:i=1,perPage:a=20,enrichments:o=[],group:s=[],jsonAttributes:c={}},l,u={})=>{let d=Object.keys(l.rawAttributes),f=Object.keys(l?.associations||{});return!n||n.length===0?n=d:B(n,d),z(t,d,f,u),H(e,d,f,u.additionalAllowedAttributes),pe(o,u),V(c,d),Array.isArray(s)||S(`group must be an array`),r.length&&typeof r==`object`?he(r,l?.associations):r&&typeof r!=`object`&&S(`include must be an array`),me({page:i,perPage:a}),!0},{object:W,string:G,number:K,any:ge,array:q,alternatives:_e}=o.types(),ve=n(),ye=W.keys({query:W,attributes:q.items(G),order:q.items(G),page:K,perPage:K,include:q.items(ge),searchTerm:G,group:q.items(G),enrichments:_e.try(q.items(G),W.pattern(G,{exclude:q.items(G)})),jsonAttributes:o.object({select:o.array().items(o.object({columnName:o.string().required(),keys:o.array().items(o.string().required()).required(),alias:o.string().optional()})).default([]),computed:o.array().items(o.object({columnName:o.string().required(),action:o.string().valid(...Object.values(E)).required(),alias:o.string().required()})).default([])}).default({})}),J=(e,t,n={})=>{let{query:i,attributes:a,order:o,page:s,perPage:c,include:l,group:u,enrichments:d,jsonAttributes:f}=t,p=ye.validate(t);if(p.error)throw new r([p.error]);U({query:i,attributes:a,order:o,page:s,perPage:c,include:l,enrichments:d,group:u,jsonAttributes:f},e,n)},be=(e,t={},n=`body`)=>(r,i,o)=>{try{J(e,r[n],t),o()}catch(e){let{query:o,attributes:s,order:c}=r[n];a(e,i,{logger:t.logger??ve,message:`error in query middleware`,payload:{error:e,query:o,attributes:s,order:c}})}},Y=(e,t,n={})=>{let{order:r,page:i,perPage:a,include:o,query:s,attributes:c,searchTerm:l,jsonAttributes:u}=t,{query:d,externalQueryValues:f,order:p,page:m,perPage:h,include:g,scopes:_,attributes:v}=de({query:s,order:r,page:i,perPage:a,include:o,attributes:c,searchTerm:l,jsonAttributes:u},e,n);t.query=d,t.externalQueryValues=f,t.order=p,t.attributes=v,t.page=m,t.perPage=h,t.include=g,t.scopes=_,n.includeRawPayload&&(t.rawPayload={order:r,page:i,perPage:a,include:o,query:s,attributes:c,searchTerm:l})},xe=(e,t={},n=`body`)=>(r,i,a)=>{Y(e,r[n],t),a()},Se=({model:e,logger:t,validationOptions:n,formatOptions:r,modelName:o=e.constructor?.name,additionalScopes:s=[],modifyQueryValues:c,onRowsRetrieved:l})=>async(u,d)=>{try{J(e,u.body,{...n,logger:t})}catch(e){a(e,d,{logger:t,message:`error in query endpoint`,payload:T(u.body,[`query`,`order`,`attributes`])});return}try{Y(e,u.body,r);let n=Object.assign(T(u.body,[`query`,`externalQueryValues`,`order`,`attributes`,`page`,`perPage`,`include`,`scopes`,`enrichments`]),{distinct:!0});t.info(`querying ${o}`,{queryValues:n});let i=c?.(n)??n,{scopes:a=[],query:f,perPage:p,page:m,enrichments:h,externalQueryValues:g,..._}=i,v=await e.scope([...s,...a]).findAndCountAll({where:f,limit:p,offset:(m-1)*p,..._});if(!v.rows.length||!l){d.json(v);return}let y=await l(v,i);d.json(y)}catch(e){a(new i(e),d,{logger:t,message:`Error while querying ${o}`,payload:{query:u.body}})}};var X=class e extends Error{constructor(e,t=!1){super(e),this.name=`BulkerError`,this.retryable=t,Object.setPrototypeOf(this,new.target.prototype)}static isBulkerError(t){return t instanceof e}static wrap(t,n,r=!1){return t instanceof e?new e(n??t.message,r||t.retryable):new e(n?`${n}: ${t instanceof Error?t.message:String(t)}`:String(t),r)}static retryable(t){return new e(t,!0)}static nonRetryable(t){return new e(t,!1)}};const Ce=e=>e.length===0?`No keys provided`:e.length===1?`Key "${e[0]}" is required`:`Exactly one of [${e.join(`, `)}] must be provided`;function we(e,t){return p.object(e).refine(e=>t.filter(t=>e[t]!==void 0&&e[t]!==null).length===1,{message:Ce(t)})}const Z=p.string().uuid();function Q(e){return we(Object.fromEntries(e.map(e=>[e,p.union([Z,p.array(Z)]).optional()])),e)}const Te=[`businessModelId`,`fleetId`,`demandSourceId`,`contextId`,`userId`,`businessAccountId`,`activeBusinessModelId`];var Ee=class{constructor(e,t){if(this.bulker=e,this.opts=t,this.rabbitQueueName=null,this.rabbit=null,this.bulkHandler=async(e,t,n)=>{try{let{query:n,payload:r,preview:i}=e.body??{},a=r;if(r&&this.payloadSchema)try{a=this.payloadSchema.parse(r)}catch(e){if(e instanceof p.ZodError)return t.status(400).json({error:`invalid_payload`,details:e.issues.map(e=>`${e.path.join(`.`)}: ${e.message}`).join(`, `)});throw e}try{J(this.model,n,{logger:this.bulker.logger})}catch(e){return t.status(400).json({error:`invalid_query`,details:e.message||e})}let o=this.identityScopeSchema.safeParse(n?.query||{});if(!o.success)return t.status(400).json({error:`invalid_identity_scope`,details:o.error.issues.map(e=>`${e.path.join(`.`)}: ${e.message}`).join(`, `)});Y(this.model,n,{includeRawPayload:!0});let{query:s,...c}=Object.assign(T(n,[`query`,`externalQueryValues`,`order`,`include`,`scopes`,`enrichments`]),{distinct:!0}),u=[];if(this.opts.additionalIdsHook){let{rawPayload:e}=n;try{u=await this.opts.additionalIdsHook({rawPayload:e,payload:a}),this.bulker.logger.info(`additionalIdsHook returned ${u.length} IDs for action ${this.action}`)}catch(e){return this.bulker.logger.error(`Error in additionalIdsHook for action ${this.action}: ${e.message||e}`,{err:e}),t.status(500).json({error:`additional_ids_hook_error`,details:e.message||e})}}let d=[];s&&typeof s==`object`&&Object.keys(s).length>0&&d.push(s),Array.isArray(u)&&u.length>0&&d.push({id:{$in:u}});let f;if(d.length===0)return t.status(400).json({error:`no_query`,details:`No valid query provided to select records`});if(d.length===1){let[e]=d;f=e}else f={$or:d};this.bulker.logger.info(`Constructed final where clause for action ${this.action}`,{where:f});let m={where:f,...c},h=await this.model.scope(this.modelScopes).count({...m,col:this.idField});if(i)return t.json({estimatedCount:h});let g=l();return await this.bulker.jobManager.initJob(g,{status:`queued`,total:h,action:this.action}),this.bulker.emitEvent(`job:created`,{jobId:g,action:this.action,total:h}),setImmediate(()=>{this.rabbitScanAndEnqueue(g,m,a).catch(e=>this.bulker.logger.error(e))}),t.status(202).json({jobId:g,estimatedCount:h})}catch(e){return this.bulker.logger.error(`Error in bulkHandler for action ${this.action}: ${e.message||e}`,{err:e}),n(e)}},this.action=t.action,this.model=t.model,this.modelScopes=t.modelScopes??[],this.consumer=t.consumer,this.rabbit=e.rabbit,this.rabbitQueueName=t.rabbitQueueName??`bulk-${this.action}-queue`,this.consumerOptions={enableRabbitTrace:!0,...this.opts.consumerOptions},this.payloadSchema=t.payloadSchema,!/^[a-zA-Z0-9-_]+$/.test(this.action))throw Error(`BulkRoute action must be alphanumeric`);if(!this.model||typeof this.model.findAll!=`function`||typeof this.model.count!=`function`)throw Error(`BulkRoute model must be a valid Sequelize model`);if(typeof this.consumer!=`function`)throw Error(`BulkRoute consumer must be a function`);if(this.payloadSchema&&!(this.payloadSchema instanceof p.ZodType))throw Error(`BulkRoute payloadSchema must be a Zod schema`);if(this.modelScopes&&!Array.isArray(this.modelScopes))throw Error(`BulkRoute scopes must be an array of strings`);if(this.opts.additionalIdsHook&&typeof this.opts.additionalIdsHook!=`function`)throw Error(`BulkRoute additionalIdsHook must be a function`);this.idField=t.idField??e.defaults.idField,this.pageSize=t.pageSize??e.defaults.pageSize,this.consumerBatchSize=t.consumerBatchSize??e.defaults.consumerBatchSize;let n=this.model.rawAttributes||{},r=(t.identityScopes&&t.identityScopes.length>0?t.identityScopes:Te).filter(e=>!!n[e]);r.length===0&&this.bulker.logger.warn(`BulkRoute for action ${this.action} has no valid identityScopes configured - all records will be accessible`),this.bulker.logger.info(`BulkRoute for action ${this.action} using idField ${this.idField}, pageSize ${this.pageSize}, consumerBatchSize ${this.consumerBatchSize}, identityScopes: ${r.join(`, `)}`),this.identityScopeSchema=Q(r),this.startRabbitWorker().catch(e=>{this.bulker.logger.error(`Failed to start RabbitMQ worker for queue ${this.rabbitQueueName}: ${e.message||e}`)})}async getUserJobs(e,t=20){return this.bulker.jobManager.getUserJobs(e,t)}batchIds(e){let t=[];for(let n=0;n<e.length;n+=this.consumerBatchSize)t.push(e.slice(n,n+this.consumerBatchSize));return t}async rabbitScanAndEnqueue(t,n,r){if(!this.rabbit)throw Error(`RabbitMQ not configured in Bulker`);let i=Date.now().toString();if(this.bulker.getUserId){let e=this.bulker.getUserId();e&&await this.bulker.jobManager.addUserJob(e,t)}await this.bulker.jobManager.setJobFields(t,{status:`running`,startTime:i}),this.bulker.emitEvent(`job:started`,{jobId:t,action:this.action});let a=null,o=0,s=0;for(;;){let i=await this.bulker.jobManager.getJobField(t,`status`);if(!i||i===`canceled`){this.bulker.logger.info(`Job ${t} was canceled, stopping scan and enqueue`);break}let c=a?{[e.and]:[n.where,{[this.idField]:{[e.gt]:a}}]}:n.where,l=await this.model.scope(this.modelScopes).findAll({where:c,...T(n,[`include`,`order`,`scopes`]),attributes:[this.idField],order:[[this.idField,`ASC`]],limit:this.pageSize,raw:!0,subQuery:!1});if(l.length===0)break;let u,d=l.map(e=>e[this.idField]);this.consumerBatchSize===1&&d.length===1&&(u=d[0]);let f=this.batchIds(d).map(e=>({jobId:t,ids:e,payload:r,id:u}));this.bulker.logger.info(`Enqueuing ${f.length} batched messages (${d.length} total IDs) to RabbitMQ queue ${this.rabbitQueueName}`);let p=(await Promise.allSettled(f.map(e=>this.rabbit.sendToQueue(this.rabbitQueueName,e)))).filter(e=>e.status===`rejected`);if(p.length>0)throw this.bulker.logger.error(`Failed to enqueue ${p.length} messages to RabbitMQ`,{rejected:p}),Error(`Failed to enqueue ${p.length} messages to RabbitMQ`);o+=l.length,await this.bulker.jobManager.incrJobField(t,`queued`,l.length),a=l[l.length-1][this.idField],s+=1,this.bulker.emitEvent(`scan:page`,{jobId:t,action:this.action,pageNumber:s,itemsInPage:l.length});let m=await this.bulker.jobManager.getJob(t);m&&this.bulker.emitEvent(`job:queued`,{jobId:t,action:this.action,queued:o,total:m.total})}o===0&&await this.bulker.jobManager.completeEmptyJob(t)}async startRabbitWorker(){return this.bulker.logger.info(`Starting RabbitMQ consumer for queue ${this.rabbitQueueName}`),this.bulker.emitEvent(`worker:started`,{action:this.action,queueName:this.rabbitQueueName}),this.rabbit?.consume(this.rabbitQueueName,async(e,t,n)=>{if(!e)return;let{jobId:r,ids:i,payload:a}=e.content,o=!1,s=async()=>{if(!o){o=!0,await Promise.all([this.bulker.jobManager.ack(r,i.length),t()]);for(let e of i)this.bulker.emitEvent(`item:processed`,{jobId:r,action:this.action,id:e})}},c=async(e,t)=>{if(o)return;o=!0;let s=t;s===void 0&&(s=X.isBulkerError(e)?e.retryable:!1),await Promise.all([this.bulker.jobManager.nack(r,e?e.message||String(e):`nacked`,{ids:i,payload:a},i.length),n(void 0,{skipRetry:!s})]);for(let t of i)s?this.bulker.emitEvent(`item:retrying`,{jobId:r,action:this.action,id:t}):this.bulker.emitEvent(`item:failed`,{jobId:r,action:this.action,id:t,error:e?e.message||String(e):`nacked`})},l=await this.bulker.jobManager.getJobField(r,`status`);if(!l||l===`canceled`){await this.bulker.jobManager.incrJobField(r,`processed`,i.length);return}try{this.bulker.logger.info(`Processing job ${r} action ${this.action} ids ${i.join(`,`)}`);for(let e of i)this.bulker.emitEvent(`item:processing`,{jobId:r,action:this.action,id:e});await this.consumer({jobId:r,ids:i,id:this.consumerBatchSize===1?i[0]:void 0,payload:a},s,c),await s()}catch(e){this.bulker.logger.error(`Error processing job ${r} action ${this.action} ids ${i.join(`,`)}: ${e.message||e}`,{err:e}),this.bulker.emitEvent(`worker:error`,{action:this.action,error:e.message||String(e)}),await c(e,!1)}},this.consumerOptions)}},De=class{constructor(e,t,n=`/bulk-actions`){this.routes=[],this.actionsToRouteMap=new Map,this.bulkHandler=async(e,t,n)=>{try{let{action:r}=e.body??{};if(!r)return t.status(400).json({error:`missing_action`,message:`Request body must include an "action" field`});let i=this.actionsToRouteMap.get(r);return i?i.bulkHandler(e,t,n):t.status(404).json({error:`unknown_action`,message:`Action "${r}" is not registered`,availableActions:Array.from(this.actionsToRouteMap.keys())})}catch(e){return this.bulker.logger.error(`Error in BulkRouter bulkHandler: ${e.message||e}`,{err:e}),n(e),null}},this.getJobHandler=async(e,t,n)=>{try{if(!e.params.id)return t.status(400).json({error:`missing_id`});let n=await this.bulker.jobManager.getJob(e.params.id);return n?t.json(n):t.status(404).json({error:`not_found`})}catch(e){return n(e)}},this.cancelJobHandler=async(e,t,n)=>{try{return e.params.id?await this.bulker.jobManager.cancelJob(e.params.id)?(this.bulker.logger.info(`Job ${e.params.id} cancel requested`),t.json({ok:!0})):t.status(404).json({error:`not_found`}):t.status(400).json({error:`missing_id`})}catch(e){return n(e)}},this.getMyJobsHandler=async(e,t,n)=>{try{if(!this.bulker.getUserId)return t.status(400).json({error:`user_id_function_not_configured`,message:`Bulker instance does not have a getUserId function configured`});let e=this.bulker.getUserId();if(!e)return t.json([]);let n=(await this.bulker.jobManager.getUserJobs(e)).filter(e=>e!==null);return n.sort((e,t)=>{let n=e.createdAt?new Date(e.createdAt).getTime():0;return(t.createdAt?new Date(t.createdAt).getTime():0)-n}),t.json(n)}catch(e){return n(e)}},this.bulker=e,this.router=t??d(),this.staticRoute=n,this.registerStaticRoutes()}registerStaticRoutes(){this.router.post(this.staticRoute,this.bulkHandler),this.router.get(`/jobs/:id`,this.getJobHandler),this.router.get(`/jobs`,this.getMyJobsHandler),this.router.post(`/jobs/:id/cancel`,this.cancelJobHandler)}addAction(e,t){let n=new Ee(this.bulker,{action:e,...t});if(this.bulker.logger.info(`Registering action handler: ${e}`),this.actionsToRouteMap.has(e))throw Error(`Action "${e}" is already registered`);return this.actionsToRouteMap.set(e,n),this.routes.push(n),n}getRouter(){return this.router}},$=class e{constructor(e,t){this.redis=e,this.defaults={maxJobsPerUser:t?.maxJobsPerUser??-1,jobTtlSeconds:t?.jobTtlSeconds??168*3600,errorLogLimit:t?.errorLogLimit??10}}setEventEmitter(e){this.eventEmitter=e}static jobKey(e){return`bulker:job:${e}`}static userJobsKey(e){return`bulker:user:${e}:jobs`}async initJob(t,n){let r=Date.now().toString(),i=e.jobKey(t),a=this.redis.multi();a.hSet(i,{status:n.status,total:String(n.total),queued:`0`,processed:`0`,succeeded:`0`,failed:`0`,action:n.action,errors:JSON.stringify([]),createdAt:r,updatedAt:r,startTime:r,endTime:``}),a.expire(i,this.defaults.jobTtlSeconds),await a.exec()}async getJob(t){let n=e.jobKey(t),r=await this.redis.hGetAll(n);if(!r||Object.keys(r).length===0)return null;let i=r.startTime?Number(r.startTime):null,a=r.endTime&&r.endTime!==``?Number(r.endTime):null,o=Date.now(),s={startTime:i?new Date(i).toISOString():null,endTime:a?new Date(a).toISOString():null,durationMs:i?(a||o)-i:null};return{jobId:t,status:r.status,action:r.action,total:Number(r.total??0),queued:Number(r.queued??0),processed:Number(r.processed??0),succeeded:Number(r.succeeded??0),failed:Number(r.failed??0),errors:r.errors?JSON.parse(r.errors):[],createdAt:r.createdAt?new Date(Number(r.createdAt)).toISOString():void 0,updatedAt:r.updatedAt?new Date(Number(r.updatedAt)).toISOString():void 0,duration:s}}async setJobField(t,n,r){let i=e.jobKey(t);if(!await this.redis.exists(i))return!1;let a=this.redis.multi();return a.hSet(i,n,r),a.hSet(i,`updatedAt`,Date.now().toString()),await a.exec(),!0}async setJobFields(t,n){let r=e.jobKey(t);if(!await this.redis.exists(r))return!1;let i=this.redis.multi();return i.hSet(r,n),i.hSet(r,`updatedAt`,Date.now().toString()),await i.exec(),!0}async incrJobField(t,n,r=1){let i=e.jobKey(t),a=this.redis.multi();a.hIncrBy(i,n,r),a.hSet(i,`updatedAt`,Date.now().toString());let o=await a.exec();return o?.[0]&&o[0][0]===null?o[0][1]:0}async getJobField(t,n){let r=e.jobKey(t);return this.redis.hGet(r,n)}async cancelJob(e){return this.setJobField(e,`status`,`canceled`)}async completeEmptyJob(e){let t=Date.now().toString();await this.setJobFields(e,{status:`completed`,endTime:t})}async ack(t,n=1){let r=e.jobKey(t);await this.redis.eval(`
 local jobKey = KEYS[1]
-local updatedAt = ARGV[1]
+local count = tonumber(ARGV[1]) or 1
+local updatedAt = ARGV[2]
 local total = tonumber(redis.call('HGET', jobKey, 'total'))
 if not total then
   return 0
 end
 
-local processed = redis.call('HINCRBY', jobKey, 'processed', 1)
-redis.call('HINCRBY', jobKey, 'succeeded', 1)
+local processed = redis.call('HINCRBY', jobKey, 'processed', count)
+redis.call('HINCRBY', jobKey, 'succeeded', count)
 redis.call('HSET', jobKey, 'updatedAt', updatedAt)
 
 if processed >= total then
@@ -16,37 +17,49 @@ if processed >= total then
 end
 
 return processed
-`,{keys:[n],arguments:[Date.now().toString()]});let r=await this.getJob(t);if(r&&r.status===`completed`){let e=r.duration.durationMs||0;this.eventEmitter?.emitEvent(`job:completed`,{jobId:t,action:r.action,processed:r.processed,failed:r.failed,duration:e})}}async nack(t,n,r){let i=e.jobKey(t);await this.redis.eval(`
+`,{keys:[r],arguments:[n.toString(),Date.now().toString()]});let i=await this.getJob(t);if(i&&i.status===`completed`){let e=i.duration.durationMs||0;this.eventEmitter?.emitEvent(`job:completed`,{jobId:t,action:i.action,processed:i.processed,failed:i.failed,duration:e})}}async nack(t,n,r,i=1){let a=e.jobKey(t);await this.redis.eval(`
 local jobKey = KEYS[1]
 local errorMsg = ARGV[1]
 local errorData = ARGV[2]
 local errorLogLimit = tonumber(ARGV[3])
-local updatedAt = ARGV[4]
+local count = tonumber(ARGV[4]) or 1
+local updatedAt = ARGV[5]
 
 local total = tonumber(redis.call('HGET', jobKey, 'total'))
 if not total then
   return 0
 end
 
--- Add error
+-- Add errors (one per failed item in the batch)
 local errorsJson = redis.call('HGET', jobKey, 'errors') or '[]'
 local errors = cjson.decode(errorsJson)
+local errorData = cjson.decode(errorData)
 
-if #errors >= errorLogLimit then
-  -- Keep only the last (errorLogLimit - 1) entries
-  local newErrors = {}
-  for i = #errors - errorLogLimit + 2, #errors do
-    table.insert(newErrors, errors[i])
+-- Add one error entry per failed item
+for i = 1, count do
+  if #errors >= errorLogLimit then
+    -- Keep only the last (errorLogLimit - 1) entries
+    local newErrors = {}
+    for j = #errors - errorLogLimit + 2, #errors do
+      table.insert(newErrors, errors[j])
+    end
+    errors = newErrors
   end
-  errors = newErrors
+
+  -- Add error with individual ID if ids array is provided, otherwise use batch data
+  local errorEntry = { message = errorMsg, data = errorData }
+  if errorData.ids and errorData.ids[i] then
+    errorEntry.data = { id = errorData.ids[i], payload = errorData.payload }
+  end
+
+  table.insert(errors, errorEntry)
 end
 
-table.insert(errors, { message = errorMsg, data = cjson.decode(errorData) })
 redis.call('HSET', jobKey, 'errors', cjson.encode(errors))
 
 -- Update counters
-redis.call('HINCRBY', jobKey, 'failed', 1)
-local processed = redis.call('HINCRBY', jobKey, 'processed', 1)
+redis.call('HINCRBY', jobKey, 'failed', count)
+local processed = redis.call('HINCRBY', jobKey, 'processed', count)
 redis.call('HSET', jobKey, 'updatedAt', updatedAt)
 
 if processed >= total then
@@ -55,5 +68,5 @@ if processed >= total then
 end
 
 return processed
-`,{keys:[i],arguments:[n,JSON.stringify(r),String(this.defaults.errorLogLimit),Date.now().toString()]});let a=await this.getJob(t);if(a&&a.status===`completed`){let e=a.duration.durationMs||0;this.eventEmitter?.emitEvent(`job:completed`,{jobId:t,action:a.action,processed:a.processed,failed:a.failed,duration:e})}}async addUserJob(t,n){let r=e.userJobsKey(t),i=this.redis.multi();i.lPush(r,n),this.defaults.maxJobsPerUser>0&&i.lTrim(r,0,this.defaults.maxJobsPerUser-1),i.expire(r,3600),await i.exec()}async removeUserJob(t,n){let r=e.userJobsKey(t);await this.redis.lRem(r,0,n)}async getUserJobs(t,n=20){let r=e.userJobsKey(t),i=await this.redis.lRange(r,0,n-1);return await Promise.all(i.map(e=>this.getJob(e)))}};const Oe={emitEvent:()=>{}};var ke=class extends m{constructor(e){if(super(),this.bulkRouters=[],this.sequelize=e.sequelize,this.logger=e.logger,this.rabbit=e.rabbit,this.eventsEnabled=e.emitEvents??!1,e.redis&&typeof e.redis.connect==`function`)this.redis=e.redis,this.redis.isOpen||this.redis.connect().catch(e=>{this.logger.error(`Error connecting to Redis:`,e)});else{let t=e.redis;this.redis=u({socket:{host:t.host,port:t.port},commandsQueueMaxLength:5e3,disableOfflineQueue:!1,password:t.password,database:t.db}),this.redis.connect().catch(e=>{this.logger.error(`Error connecting to Redis:`,e)})}this.getUserId=e.getUserId??(()=>null),this.defaults={maxJobsPerUser:e.defaults?.maxJobsPerUser??-1,pageSize:e.defaults?.pageSize??1e3,idField:e.defaults?.idField??`id`,workerConcurrency:e.defaults?.workerConcurrency??16,jobTtlSeconds:e.defaults?.jobTtlSeconds??168*3600,errorLogLimit:e.defaults?.errorLogLimit??10},this.jobManager=new $(this.redis,{maxJobsPerUser:this.defaults.maxJobsPerUser,jobTtlSeconds:this.defaults.jobTtlSeconds,errorLogLimit:this.defaults.errorLogLimit}),this.jobManager.setEventEmitter(this.eventsEnabled?{emitEvent:this.emitEvent.bind(this)}:Oe)}pingRedis(){return this.redis.ping()}createBulkRouter(e,t){let n=new De(this,e,t);return this.bulkRouters.push(n),n}emitEvent(e,...t){this.eventsEnabled&&this.emit(e,...t)}};export{ke as Bulker,X as BulkerError,$ as BulkerJobManager,_ as formatOperators,P as generateFilterReplacements,xe as queryFormatMiddleware,Se as queryHandler,be as queryValidationMiddleware,U as validatePayload,f as z};
+`,{keys:[a],arguments:[n,JSON.stringify(r),String(this.defaults.errorLogLimit),i.toString(),Date.now().toString()]});let o=await this.getJob(t);if(o&&o.status===`completed`){let e=o.duration.durationMs||0;this.eventEmitter?.emitEvent(`job:completed`,{jobId:t,action:o.action,processed:o.processed,failed:o.failed,duration:e})}}async addUserJob(t,n){let r=e.userJobsKey(t),i=this.redis.multi();i.lPush(r,n),this.defaults.maxJobsPerUser>0&&i.lTrim(r,0,this.defaults.maxJobsPerUser-1),i.expire(r,3600),await i.exec()}async removeUserJob(t,n){let r=e.userJobsKey(t);await this.redis.lRem(r,0,n)}async getUserJobs(t,n=20){let r=e.userJobsKey(t),i=await this.redis.lRange(r,0,n-1);return await Promise.all(i.map(e=>this.getJob(e)))}};const Oe={emitEvent:()=>{}};var ke=class extends m{constructor(e){if(super(),this.bulkRouters=[],this.sequelize=e.sequelize,this.logger=e.logger,this.rabbit=e.rabbit,this.eventsEnabled=e.emitEvents??!1,e.redis&&typeof e.redis.connect==`function`)this.redis=e.redis,this.redis.isOpen||this.redis.connect().catch(e=>{this.logger.error(`Error connecting to Redis:`,e)});else{let t=e.redis;this.redis=u({socket:{host:t.host,port:t.port},commandsQueueMaxLength:5e3,disableOfflineQueue:!1,password:t.password,database:t.db}),this.redis.connect().catch(e=>{this.logger.error(`Error connecting to Redis:`,e)})}this.getUserId=e.getUserId??(()=>null),this.defaults={maxJobsPerUser:e.defaults?.maxJobsPerUser??-1,pageSize:e.defaults?.pageSize??1e3,idField:e.defaults?.idField??`id`,consumerBatchSize:e.defaults?.consumerBatchSize??1,workerConcurrency:e.defaults?.workerConcurrency??16,jobTtlSeconds:e.defaults?.jobTtlSeconds??168*3600,errorLogLimit:e.defaults?.errorLogLimit??10},this.jobManager=new $(this.redis,{maxJobsPerUser:this.defaults.maxJobsPerUser,jobTtlSeconds:this.defaults.jobTtlSeconds,errorLogLimit:this.defaults.errorLogLimit}),this.jobManager.setEventEmitter(this.eventsEnabled?{emitEvent:this.emitEvent.bind(this)}:Oe)}pingRedis(){return this.redis.ping()}createBulkRouter(e,t){let n=new De(this,e,t);return this.bulkRouters.push(n),n}emitEvent(e,...t){this.eventsEnabled&&this.emit(e,...t)}};export{ke as Bulker,X as BulkerError,$ as BulkerJobManager,_ as formatOperators,P as generateFilterReplacements,xe as queryFormatMiddleware,Se as queryHandler,be as queryValidationMiddleware,U as validatePayload,f as z};
 //# sourceMappingURL=index.js.map