@autofleet/sheilta 2.11.1-beta.2 → 2.11.1-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/index.cjs +4 -4
- package/lib/index.cjs.map +1 -1
- package/lib/index.d.cts +35 -13
- package/lib/index.d.ts +35 -13
- package/lib/index.js +3 -3
- package/lib/index.js.map +1 -1
- package/package.json +1 -1
package/lib/index.d.cts
CHANGED
|
@@ -13,17 +13,22 @@ declare const formatOperators: (sequelize?: {
|
|
|
13
13
|
}) => Record<string, symbol>;
|
|
14
14
|
//#endregion
|
|
15
15
|
//#region src/formatter/jsonAttributesFormater.d.ts
|
|
16
|
-
declare const
|
|
16
|
+
declare const ComputedActions: {
|
|
17
17
|
readonly length: "length";
|
|
18
|
-
readonly timezoneConvert: "timezoneConvert";
|
|
19
|
-
readonly selectFromJson: "selectFromJson";
|
|
20
18
|
};
|
|
21
|
-
interface
|
|
19
|
+
interface JsonSelectAttribute {
|
|
22
20
|
columnName: string;
|
|
23
|
-
|
|
21
|
+
keys: string[];
|
|
22
|
+
alias?: string;
|
|
23
|
+
}
|
|
24
|
+
interface JsonComputedAttribute {
|
|
25
|
+
columnName: string;
|
|
26
|
+
action: keyof typeof ComputedActions;
|
|
24
27
|
alias: string;
|
|
25
|
-
|
|
26
|
-
|
|
28
|
+
}
|
|
29
|
+
interface JsonAttributes {
|
|
30
|
+
select?: JsonSelectAttribute[];
|
|
31
|
+
computed?: JsonComputedAttribute[];
|
|
27
32
|
}
|
|
28
33
|
//#endregion
|
|
29
34
|
//#region src/formatter/index.d.ts
|
|
@@ -50,13 +55,19 @@ type ConditionValue = ConditionWithOperator | ConditionWithOperator[] | string |
|
|
|
50
55
|
* @returns The replacements object.
|
|
51
56
|
*/
|
|
52
57
|
declare const generateFilterReplacements: (conditions: Record<string, ConditionValue>) => Record<string, string>;
|
|
58
|
+
interface CustomWhereClause {
|
|
59
|
+
alias: string;
|
|
60
|
+
action: string;
|
|
61
|
+
columnName: string;
|
|
62
|
+
additionalPayload?: Record<string, any>;
|
|
63
|
+
}
|
|
53
64
|
interface Include {
|
|
54
65
|
association?: string;
|
|
55
66
|
model?: string;
|
|
56
67
|
required?: boolean;
|
|
57
68
|
include?: Include[];
|
|
58
|
-
|
|
59
|
-
|
|
69
|
+
where?: any;
|
|
70
|
+
customWhereClauses?: CustomWhereClause[];
|
|
60
71
|
}
|
|
61
72
|
interface FormatPayloadData {
|
|
62
73
|
order?: string[];
|
|
@@ -66,7 +77,7 @@ interface FormatPayloadData {
|
|
|
66
77
|
query?: Record<string, unknown>;
|
|
67
78
|
attributes?: string[] | null;
|
|
68
79
|
searchTerm?: string | null;
|
|
69
|
-
|
|
80
|
+
jsonAttributes?: JsonAttributes;
|
|
70
81
|
subQuery?: boolean;
|
|
71
82
|
}
|
|
72
83
|
type Literal = ReturnType<typeof literal>;
|
|
@@ -96,7 +107,7 @@ declare const formatPayload: ({
|
|
|
96
107
|
query,
|
|
97
108
|
attributes,
|
|
98
109
|
searchTerm,
|
|
99
|
-
|
|
110
|
+
jsonAttributes,
|
|
100
111
|
subQuery
|
|
101
112
|
}: FormatPayloadData, model?: any, options?: FormatPayloadOptions) => FormattedPayload;
|
|
102
113
|
//#endregion
|
|
@@ -133,7 +144,18 @@ interface PayloadValidationData {
|
|
|
133
144
|
};
|
|
134
145
|
};
|
|
135
146
|
group?: string[];
|
|
136
|
-
|
|
147
|
+
jsonAttributes?: {
|
|
148
|
+
select?: {
|
|
149
|
+
columnName: string;
|
|
150
|
+
keys: string[];
|
|
151
|
+
alias?: string;
|
|
152
|
+
}[];
|
|
153
|
+
computed?: {
|
|
154
|
+
columnName: string;
|
|
155
|
+
action: "length";
|
|
156
|
+
alias: string;
|
|
157
|
+
}[];
|
|
158
|
+
};
|
|
137
159
|
subQuery?: boolean;
|
|
138
160
|
}
|
|
139
161
|
declare const validatePayload: ({
|
|
@@ -145,7 +167,7 @@ declare const validatePayload: ({
|
|
|
145
167
|
perPage,
|
|
146
168
|
enrichments,
|
|
147
169
|
group,
|
|
148
|
-
|
|
170
|
+
jsonAttributes,
|
|
149
171
|
subQuery
|
|
150
172
|
}: PayloadValidationData, model?: any, options?: MiddlewareValidationOption) => boolean;
|
|
151
173
|
//#endregion
|
package/lib/index.d.ts
CHANGED
|
@@ -13,17 +13,22 @@ declare const formatOperators: (sequelize?: {
|
|
|
13
13
|
}) => Record<string, symbol>;
|
|
14
14
|
//#endregion
|
|
15
15
|
//#region src/formatter/jsonAttributesFormater.d.ts
|
|
16
|
-
declare const
|
|
16
|
+
declare const ComputedActions: {
|
|
17
17
|
readonly length: "length";
|
|
18
|
-
readonly timezoneConvert: "timezoneConvert";
|
|
19
|
-
readonly selectFromJson: "selectFromJson";
|
|
20
18
|
};
|
|
21
|
-
interface
|
|
19
|
+
interface JsonSelectAttribute {
|
|
22
20
|
columnName: string;
|
|
23
|
-
|
|
21
|
+
keys: string[];
|
|
22
|
+
alias?: string;
|
|
23
|
+
}
|
|
24
|
+
interface JsonComputedAttribute {
|
|
25
|
+
columnName: string;
|
|
26
|
+
action: keyof typeof ComputedActions;
|
|
24
27
|
alias: string;
|
|
25
|
-
|
|
26
|
-
|
|
28
|
+
}
|
|
29
|
+
interface JsonAttributes {
|
|
30
|
+
select?: JsonSelectAttribute[];
|
|
31
|
+
computed?: JsonComputedAttribute[];
|
|
27
32
|
}
|
|
28
33
|
//#endregion
|
|
29
34
|
//#region src/formatter/index.d.ts
|
|
@@ -50,13 +55,19 @@ type ConditionValue = ConditionWithOperator | ConditionWithOperator[] | string |
|
|
|
50
55
|
* @returns The replacements object.
|
|
51
56
|
*/
|
|
52
57
|
declare const generateFilterReplacements: (conditions: Record<string, ConditionValue>) => Record<string, string>;
|
|
58
|
+
interface CustomWhereClause {
|
|
59
|
+
alias: string;
|
|
60
|
+
action: string;
|
|
61
|
+
columnName: string;
|
|
62
|
+
additionalPayload?: Record<string, any>;
|
|
63
|
+
}
|
|
53
64
|
interface Include {
|
|
54
65
|
association?: string;
|
|
55
66
|
model?: string;
|
|
56
67
|
required?: boolean;
|
|
57
68
|
include?: Include[];
|
|
58
|
-
|
|
59
|
-
|
|
69
|
+
where?: any;
|
|
70
|
+
customWhereClauses?: CustomWhereClause[];
|
|
60
71
|
}
|
|
61
72
|
interface FormatPayloadData {
|
|
62
73
|
order?: string[];
|
|
@@ -66,7 +77,7 @@ interface FormatPayloadData {
|
|
|
66
77
|
query?: Record<string, unknown>;
|
|
67
78
|
attributes?: string[] | null;
|
|
68
79
|
searchTerm?: string | null;
|
|
69
|
-
|
|
80
|
+
jsonAttributes?: JsonAttributes;
|
|
70
81
|
subQuery?: boolean;
|
|
71
82
|
}
|
|
72
83
|
type Literal = ReturnType<typeof literal>;
|
|
@@ -96,7 +107,7 @@ declare const formatPayload: ({
|
|
|
96
107
|
query,
|
|
97
108
|
attributes,
|
|
98
109
|
searchTerm,
|
|
99
|
-
|
|
110
|
+
jsonAttributes,
|
|
100
111
|
subQuery
|
|
101
112
|
}: FormatPayloadData, model?: any, options?: FormatPayloadOptions) => FormattedPayload;
|
|
102
113
|
//#endregion
|
|
@@ -133,7 +144,18 @@ interface PayloadValidationData {
|
|
|
133
144
|
};
|
|
134
145
|
};
|
|
135
146
|
group?: string[];
|
|
136
|
-
|
|
147
|
+
jsonAttributes?: {
|
|
148
|
+
select?: {
|
|
149
|
+
columnName: string;
|
|
150
|
+
keys: string[];
|
|
151
|
+
alias?: string;
|
|
152
|
+
}[];
|
|
153
|
+
computed?: {
|
|
154
|
+
columnName: string;
|
|
155
|
+
action: "length";
|
|
156
|
+
alias: string;
|
|
157
|
+
}[];
|
|
158
|
+
};
|
|
137
159
|
subQuery?: boolean;
|
|
138
160
|
}
|
|
139
161
|
declare const validatePayload: ({
|
|
@@ -145,7 +167,7 @@ declare const validatePayload: ({
|
|
|
145
167
|
perPage,
|
|
146
168
|
enrichments,
|
|
147
169
|
group,
|
|
148
|
-
|
|
170
|
+
jsonAttributes,
|
|
149
171
|
subQuery
|
|
150
172
|
}: PayloadValidationData, model?: any, options?: MiddlewareValidationOption) => boolean;
|
|
151
173
|
//#endregion
|
package/lib/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import{Op as e,literal as t}from"sequelize";import n from"@autofleet/logger";import{BadRequest as r,UnexpectedError as i,handleError as a}from"@autofleet/errors";import o from"joi";import{customFields as s}from"@autofleet/common-types";import{randomInt as c,randomUUID as l}from"node:crypto";import{createClient as u}from"redis";import{Router as d}from"express";import{z as f,z as p}from"zod";import{EventEmitter as m}from"node:events";const h=[`eq`,`ne`,`gte`,`gt`,`lte`,`lt`,`not`,`in`,`notIn`,`is`,`like`,`iLike`,`notLike`,`between`,`and`,`or`,`overlap`,`contains`],g={$eq:`=`,$ne:`!=`,$gte:`>=`,$gt:`>`,$lte:`<=`,$lt:`<`,$not:`NOT`,$in:`IN`,$notIn:`NOT IN`,$is:`IS`,$like:`LIKE`,$iLike:`ILIKE`,$notLike:`NOT LIKE`,$and:`AND`,$or:`OR`},_=Object.fromEntries(h.map(t=>[`${`$`+t}`,e[t]])),v=(t={Op:e})=>{let{Op:n}=t;return Object.fromEntries(h.map(e=>[`${`$`+e}`,n[e]]))},y=e=>`\$${e}\$`,b=(e,t)=>e.includes(`.`)&&t.includes(e.split(`.`,1)[0]),x=(e,t)=>{let n=e;return e.includes(`-`)&&([,n]=n.split(`-`,2)),b(n,t)&&([n]=n.split(`.`,1)),n},S=e=>e.includes(`-`),C=e=>{throw new r([Error(e)])},w=e=>e.split(`.`,2)[1],T=(e=5)=>Array.from({length:e},()=>`ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz`.charAt(c(52))).join(``),E=(e,t)=>Object.fromEntries(t.map(t=>[t,e[t]])),D={length:`length`,timezoneConvert:`timezoneConvert`,selectFromJson:`selectFromJson`};function O(e){return e.replace(/(?!^)[A-Z]/g,e=>`_${e.toLowerCase()}`)}function ee(e){switch(e.action){case D.length:return[t(`jsonb_array_length(${O(e.columnName)})`),e.alias];case D.timezoneConvert:{let n=e.additionalPayload?.timezoneColumn;if(!n)throw Error(`additionalPayload.timezoneColumn is required for timezoneConvert action`);return[t(`(${O(e.columnName)} AT TIME ZONE ${O(n)})`),e.alias]}case D.selectFromJson:{let n=e.additionalPayload?.keys;if(!n||!Array.isArray(n))throw Error(`additionalPayload.keys is required for selectFromJson action`);let r=O(e.columnName);return[t(`json_build_object(${n.map(e=>`'${e}', ${r} -> '${e}'`).join(`, `)})`),e.alias]}default:throw e.action,Error(`Unknown action: ${e.action}`)}}function k(e=[]){return e.map(e=>ee(e))}const A=`DESC`,j=`customFields.`,{CUSTOM_FIELDS_FILTER_SCOPE:M,CUSTOM_FIELDS_SORT_SCOPE:N}=s,P=e=>[`string`,`number`].includes(typeof e)||Array.isArray(e)?e:Object.entries(e).map(([e,t])=>({operator:g[e],value:t})),te=(e,t={})=>{let{literalAttributes:n=[],DBFormatter:r=void 0}=t,[i,a]=e.reduce((e,t)=>{let[i,a=`ASC`]=Array.isArray(t)?t:[t],o=n?.find(e=>e.attribute===i);if(o){let t=r?r(`"${o.attribute}" ${a}`):`${o.attribute} ${a}`;e[1].push(o.literal),e[0].push([t])}else e[0].push(t);return e},[[],[]]);return[i,a]},F=e=>{let t={};return Object.entries(e).forEach(([e,n])=>{let r=T();if(t[r]=e.split(j,2)[1],Array.isArray(n))n.forEach(e=>{let n=T();t[n]=typeof e==`string`?e:e.value});else if(typeof n==`string`||typeof n==`number`){let e=T();t[e]=n}else if(n?.operator){let e=T();t[e]=n.value}}),t},ne=e=>{let t={};return e.forEach(e=>{if(e.startsWith(j)){let n=T();t[n]=e.split(j,2)[1]}else if(e.substring(1).startsWith(j)){let n=T();t[n]=e.substring(1).split(j,2)[1]}}),t},re=(e,t)=>({...ne(e),...F(t)}),ie=({order:e,associationModels:t=[],replacementsMap:n={}})=>{let r=[],i=new Map;return e.forEach(e=>{if([e,e.substring(1)].some(e=>e.startsWith(j))){i.has(N)||i.set(N,{});let t=e.split(j,2)[1];i.get(N)[t]=S(e)?A:`ASC`;return}let n=[x(e,t)],a=S(e);b(a?e.split(`-`,2)[1]:e,t)&&n.push(w(e)),a&&n.push(A),r.push(n)}),{formattedOrders:r,replacementsMap:n,orderScopes:Array.from(i.entries()).map(([e,t])=>t?{method:[e,{replacementsMap:n,scopeValue:t}]}:e)}},ae=e=>e||1,I=e=>e||20,L=(e,t={})=>{let n=e.map(e=>{let n=t[typeof e==`string`?e:e.association||e.model],r=typeof e==`string`?void 0:e.attributes;if(typeof e!=`string`&&e.customAttributes){let t=k(e.customAttributes);r=[...e.attributes??[],...t]}return{...typeof e!=`string`&&e,association:n,required:typeof e==`string`||e.required!==!1,...r&&{attributes:r},...typeof e!=`string`&&e.include&&{include:L(e.include,n?.target?.associations)}}});return n=n.map(({model:e,customAttributes:t,...n})=>n),n},R=e=>{if(e==null)return e;if(Array.isArray(e))return e.map(e=>R(e));if(typeof e==`object`&&e.constructor===Object){let t={};for(let[n,r]of Object.entries(e)){let e=_[n]??n;t[e]=R(r)}return t}return e},oe=({query:e,associationModels:t,replacementsMap:n,additionalAllowedAttributes:r=[],useOpSymbols:i=!1})=>{let a={},o={},s=new Map;Object.entries(e).forEach(([e,n])=>{if(e.startsWith(j)){s.has(M)||s.set(M,{});let t=e.split(j,2)[1];s.get(M)[t]=P(n);return}if(r.includes(e)){o[e]=n;return}let i=b(e,t)?y(e):e;a[i]=n});let c=Array.from(s.entries()).map(([e,t])=>t?{method:[e,{replacementsMap:n,scopeValue:t}]}:e);return{formattedQuery:i?R(a):a,externalQueryValues:o,formattedScopes:c}},se=({searchTerm:t,attributesToSend:n,rawAttributes:r,useOpSymbols:i=!1})=>({[i?e.and:`$and`]:t.split(` `).map(t=>({[i?e.or:`$or`]:n.filter(e=>r[e].type.key===`STRING`).map(n=>({[n]:{[i?e.iLike:`$iLike`]:`%${t}%`}}))}))});var ce=({order:t=[],page:n=1,perPage:r=20,include:i=[],query:a={},attributes:o=null,searchTerm:s=null,customAttributes:c=[],subQuery:l},u,d)=>{let f=re(t,a),p=Object.keys(u?.associations||{}),{formattedOrders:m,orderScopes:h}=ie({order:[...t,`id`],associationModels:p,replacementsMap:f}),[g,_]=te(m,d),v=k(c),y=[..._,...o??[],...v],b=o?.length?y:{include:y},x=L(i,u?.associations),S=ae(n),C=I(r),w=oe({query:a,associationModels:p,replacementsMap:f,additionalAllowedAttributes:d?.additionalAllowedAttributes,useOpSymbols:d?.useOpSymbols}),{formattedScopes:T,externalQueryValues:E}=w,{formattedQuery:D}=w;if(s&&!d?.skipSearchTermFormat){let t=se({searchTerm:s,attributesToSend:o?.length?o:Object.keys(u.rawAttributes||{}),rawAttributes:u.rawAttributes,useOpSymbols:d?.useOpSymbols}),n=!Object.keys(D).length&&!Object.getOwnPropertySymbols(D).length;D=!D||n?t:{[d?.useOpSymbols?e.and:`$and`]:[D,t]}}return{query:D,order:g,page:S,perPage:C,include:x,scopes:[...T,...h],...b&&{attributes:b},...Object.keys(E).length>0&&{externalQueryValues:E},...l!==void 0&&{subQuery:l}}};const le=e=>h.includes(e.split(`$`,2)[1]),z=(e,t=[],n=[],r=[])=>{let i=e.startsWith(`$`)&&e.endsWith(`$`)?e.slice(1,-1):e;return[...t,...n].includes(i.includes(`.`)?i.split(`.`,1)[0]:i)||r.includes(i)},ue=(e,t,n,r={})=>{let i=S(e);i&&!e.startsWith(`-`)&&C(`- must be only at the beginning of the word`);let a=i?e.split(`-`,2)[1]:e,o=b(a,n),s=x(e,n),c=r?.literalAttributes?.map(e=>e.attribute)?.includes(a);!o&&s.includes(`.`)&&([s]=s.split(`.`,1)),t.includes(s)||o||c||C(`${e} is invalid. isLiteralAttribute: ${c}`)},de=(e,t)=>{t.includes(e)||C(`${e} is invalid`)},B=(e,t,n=[],r={})=>{e.forEach(e=>ue(e,t,n,r))},V=(e,t)=>{e.forEach(e=>de(e,t))},H=(e,t)=>{V(e.map(e=>e.columnName),t)},fe=(e,t)=>{let n=Array.isArray(e)?e:Object.keys(e);if(!n?.length)return;let r=n.find(e=>!t?.enrichmentAttributes?.includes(e));r&&C(`enrichment attribute ${r} is invalid`)},U=(e,t,n=[],r=[])=>{Object.entries(e).forEach(([e,i])=>{Array.isArray(i)?i[0]&&typeof i[0]==`object`&&i.map(e=>U(e,t,n,r)):le(e)||z(e,t,n,r)?i&&typeof i==`object`&&U(i,t,[],r):C(`invalid key: ${e}`)})},pe=({page:e,perPage:t})=>{e<1&&C(`Page must be greater than 0`),(t>100||t<1)&&C(`PerPage must be between 1 to 100`)},me=(e,t)=>{let n=Object.keys(t);e.forEach(e=>{z(e.association||e.model,n);let r=t[e.association||e.model]?.target;r||C(`model not found in associations`);let{rawAttributes:i}=r,a=Object.keys(i);e.where&&U(e.where,a),e.order&&B(e.order,a),e.attributes&&V(e.attributes,a),e.customAttributes&&H(e.customAttributes,a),[null,void 0,!0,!1].includes(e.required)||C(`include.required must be a boolean`)})},he=e=>{e===void 0||typeof e==`boolean`||C(`subQuery must be a boolean or undefined`)},W=({query:e={},order:t=[],attributes:n=[],include:r=[],page:i=1,perPage:a=20,enrichments:o=[],group:s=[],customAttributes:c=[],subQuery:l},u,d={})=>{let f=Object.keys(u.rawAttributes),p=Object.keys(u?.associations||{});return!n||n.length===0?n=f:V(n,f),B(t,f,p,d),U(e,f,p,d.additionalAllowedAttributes),fe(o,d),H(c,f),Array.isArray(s)||C(`group must be an array`),r.length&&typeof r==`object`?me(r,u?.associations):r&&typeof r!=`object`&&C(`include must be an array`),pe({page:i,perPage:a}),he(l),!0},{object:G,string:K,number:q,any:ge,array:J,alternatives:_e}=o.types(),ve=n(),ye=G.keys({query:G,attributes:J.items(K),order:J.items(K),page:q,perPage:q,include:J.items(ge),searchTerm:K,group:J.items(K),enrichments:_e.try(J.items(K),G.pattern(K,{exclude:J.items(K)})),customAttributes:o.array().items(o.object({columnName:o.string().required(),action:o.string().valid(...Object.values(D)).required(),alias:o.string().required(),additionalPayload:o.object().optional()})).default([]),subQuery:o.boolean()}),Y=(e,t,n={})=>{let{query:i,attributes:a,order:o,page:s,perPage:c,include:l,group:u,enrichments:d,customAttributes:f,subQuery:p}=t,m=ye.validate(t);if(m.error)throw new r([m.error]);W({query:i,attributes:a,order:o,page:s,perPage:c,include:l,enrichments:d,group:u,customAttributes:f,subQuery:p},e,n)},be=(e,t={},n=`body`)=>(r,i,o)=>{try{Y(e,r[n],t),o()}catch(e){let{query:o,attributes:s,order:c}=r[n];a(e,i,{logger:t.logger??ve,message:`error in query middleware`,payload:{error:e,query:o,attributes:s,order:c}})}},X=(e,t,n={})=>{let{order:r,page:i,perPage:a,include:o,query:s,attributes:c,searchTerm:l,customAttributes:u,subQuery:d}=t,{query:f,externalQueryValues:p,order:m,page:h,perPage:g,include:_,scopes:v,attributes:y,subQuery:b}=ce({query:s,order:r,page:i,perPage:a,include:o,attributes:c,searchTerm:l,customAttributes:u,subQuery:d},e,n);t.query=f,t.externalQueryValues=p,t.order=m,t.attributes=y,t.page=h,t.perPage=g,t.include=_,t.scopes=v,b!==void 0&&(t.subQuery=b),n.includeRawPayload&&(t.rawPayload={order:r,page:i,perPage:a,include:o,query:s,attributes:c,searchTerm:l,subQuery:d})},xe=(e,t={},n=`body`)=>(r,i,a)=>{X(e,r[n],t),a()},Se=({model:e,logger:t,validationOptions:n,formatOptions:r,modelName:o=e.constructor?.name,additionalScopes:s=[],modifyQueryValues:c,onRowsRetrieved:l})=>async(u,d)=>{try{Y(e,u.body,{...n,logger:t})}catch(e){a(e,d,{logger:t,message:`error in query endpoint`,payload:E(u.body,[`query`,`order`,`attributes`])});return}try{X(e,u.body,r);let n=Object.assign(E(u.body,[`query`,`externalQueryValues`,`order`,`attributes`,`page`,`perPage`,`include`,`scopes`,`enrichments`,`subQuery`]),{distinct:!0});t.info(`querying ${o}`,{queryValues:n});let i=c?.(n)??n,{scopes:a=[],query:f,perPage:p,page:m,enrichments:h,externalQueryValues:g,subQuery:_,...v}=i,y=await e.scope([...s,...a]).findAndCountAll({where:f,limit:p,offset:(m-1)*p,...v,..._!==void 0&&{subQuery:_}});if(!y.rows.length||!l){d.json(y);return}let b=await l(y,i);d.json(b)}catch(e){a(new i(e),d,{logger:t,message:`Error while querying ${o}`,payload:{query:u.body}})}};var Z=class e extends Error{constructor(e,t=!1){super(e),this.name=`BulkerError`,this.retryable=t,Object.setPrototypeOf(this,new.target.prototype)}static isBulkerError(t){return t instanceof e}static wrap(t,n,r=!1){return t instanceof e?new e(n??t.message,r||t.retryable):new e(n?`${n}: ${t instanceof Error?t.message:String(t)}`:String(t),r)}static retryable(t){return new e(t,!0)}static nonRetryable(t){return new e(t,!1)}};const Ce=e=>e.length===0?`No keys provided`:e.length===1?`Key "${e[0]}" is required`:`Exactly one of [${e.join(`, `)}] must be provided`;function we(e,t){return p.object(e).refine(e=>t.filter(t=>e[t]!==void 0&&e[t]!==null).length===1,{message:Ce(t)})}const Q=p.string().uuid();function Te(e){return we(Object.fromEntries(e.map(e=>[e,p.union([Q,p.array(Q)]).optional()])),e)}const Ee=[`businessModelId`,`fleetId`,`demandSourceId`,`contextId`,`userId`,`businessAccountId`,`activeBusinessModelId`];var De=class{constructor(e,t){if(this.bulker=e,this.opts=t,this.rabbitQueueName=null,this.rabbit=null,this.bulkHandler=async(e,t,n)=>{try{let{query:n,payload:r,preview:i}=e.body??{},a=r;if(r&&this.payloadSchema)try{a=this.payloadSchema.parse(r)}catch(e){if(e instanceof p.ZodError)return t.status(400).json({error:`invalid_payload`,details:e.issues.map(e=>`${e.path.join(`.`)}: ${e.message}`).join(`, `)});throw e}try{Y(this.model,n,{logger:this.bulker.logger})}catch(e){return t.status(400).json({error:`invalid_query`,details:e.message||e})}let o=this.identityScopeSchema.safeParse(n?.query||{});if(!o.success)return t.status(400).json({error:`invalid_identity_scope`,details:o.error.issues.map(e=>`${e.path.join(`.`)}: ${e.message}`).join(`, `)});X(this.model,n,{includeRawPayload:!0});let{query:s,...c}=Object.assign(E(n,[`query`,`externalQueryValues`,`order`,`include`,`scopes`,`enrichments`]),{distinct:!0}),u=[];if(this.opts.additionalIdsHook){let{rawPayload:e}=n;try{u=await this.opts.additionalIdsHook({rawPayload:e,payload:a}),this.bulker.logger.info(`additionalIdsHook returned ${u.length} IDs for action ${this.action}`)}catch(e){return this.bulker.logger.error(`Error in additionalIdsHook for action ${this.action}: ${e.message||e}`,{err:e}),t.status(500).json({error:`additional_ids_hook_error`,details:e.message||e})}}let d=[];s&&typeof s==`object`&&Object.keys(s).length>0&&d.push(s),Array.isArray(u)&&u.length>0&&d.push({id:{$in:u}});let f;if(d.length===0)return t.status(400).json({error:`no_query`,details:`No valid query provided to select records`});if(d.length===1){let[e]=d;f=e}else f={$or:d};this.bulker.logger.info(`Constructed final where clause for action ${this.action}`,{where:f});let m={where:f,...c},h=await this.model.scope(this.modelScopes).count({...m,col:this.idField});if(i)return t.json({estimatedCount:h});let g=l();return await this.bulker.jobManager.initJob(g,{status:`queued`,total:h,action:this.action}),this.bulker.emitEvent(`job:created`,{jobId:g,action:this.action,total:h}),setImmediate(()=>{this.rabbitScanAndEnqueue(g,m,a).catch(e=>this.bulker.logger.error(e))}),t.status(202).json({jobId:g,estimatedCount:h})}catch(e){return this.bulker.logger.error(`Error in bulkHandler for action ${this.action}: ${e.message||e}`,{err:e}),n(e)}},this.action=t.action,this.model=t.model,this.modelScopes=t.modelScopes??[],this.consumer=t.consumer,this.rabbit=e.rabbit,this.rabbitQueueName=t.rabbitQueueName??`bulk-${this.action}-queue`,this.consumerOptions={enableRabbitTrace:!0,...this.opts.consumerOptions},this.payloadSchema=t.payloadSchema,!/^[a-zA-Z0-9-_]+$/.test(this.action))throw Error(`BulkRoute action must be alphanumeric`);if(!this.model||typeof this.model.findAll!=`function`||typeof this.model.count!=`function`)throw Error(`BulkRoute model must be a valid Sequelize model`);if(typeof this.consumer!=`function`)throw Error(`BulkRoute consumer must be a function`);if(this.payloadSchema&&!(this.payloadSchema instanceof p.ZodType))throw Error(`BulkRoute payloadSchema must be a Zod schema`);if(this.modelScopes&&!Array.isArray(this.modelScopes))throw Error(`BulkRoute scopes must be an array of strings`);if(this.opts.additionalIdsHook&&typeof this.opts.additionalIdsHook!=`function`)throw Error(`BulkRoute additionalIdsHook must be a function`);this.idField=t.idField??e.defaults.idField,this.pageSize=t.pageSize??e.defaults.pageSize,this.consumerBatchSize=t.consumerBatchSize??e.defaults.consumerBatchSize;let n=this.model.rawAttributes||{},r=(t.identityScopes&&t.identityScopes.length>0?t.identityScopes:Ee).filter(e=>!!n[e]);r.length===0&&this.bulker.logger.warn(`BulkRoute for action ${this.action} has no valid identityScopes configured - all records will be accessible`),this.bulker.logger.info(`BulkRoute for action ${this.action} using idField ${this.idField}, pageSize ${this.pageSize},
|
|
2
|
-
consumerBatchSize ${this.consumerBatchSize}, identityScopes: ${r.join(`, `)}`),this.identityScopeSchema=
|
|
1
|
+
import{Op as e,literal as t,where as n}from"sequelize";import r from"@autofleet/logger";import{BadRequest as i,UnexpectedError as a,handleError as o}from"@autofleet/errors";import s from"joi";import{customFields as c}from"@autofleet/common-types";import{randomInt as l,randomUUID as u}from"node:crypto";import{createClient as d}from"redis";import{Router as f}from"express";import{z as p,z as m}from"zod";import{EventEmitter as h}from"node:events";const g=[`eq`,`ne`,`gte`,`gt`,`lte`,`lt`,`not`,`in`,`notIn`,`is`,`like`,`iLike`,`notLike`,`between`,`and`,`or`,`overlap`,`contains`],_={$eq:`=`,$ne:`!=`,$gte:`>=`,$gt:`>`,$lte:`<=`,$lt:`<`,$not:`NOT`,$in:`IN`,$notIn:`NOT IN`,$is:`IS`,$like:`LIKE`,$iLike:`ILIKE`,$notLike:`NOT LIKE`,$and:`AND`,$or:`OR`},v=Object.fromEntries(g.map(t=>[`${`$`+t}`,e[t]])),y=(t={Op:e})=>{let{Op:n}=t;return Object.fromEntries(g.map(e=>[`${`$`+e}`,n[e]]))},b=e=>`\$${e}\$`,x=(e,t)=>e.includes(`.`)&&t.includes(e.split(`.`,1)[0]),S=(e,t)=>{let n=e;return e.includes(`-`)&&([,n]=n.split(`-`,2)),x(n,t)&&([n]=n.split(`.`,1)),n},C=e=>e.includes(`-`),w=e=>{throw new i([Error(e)])},T=e=>e.split(`.`,2)[1],E=(e=5)=>Array.from({length:e},()=>`ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz`.charAt(l(52))).join(``),D=(e,t)=>Object.fromEntries(t.map(t=>[t,e[t]])),O={length:`length`};function k(e){return e.replace(/(?!^)[A-Z]/g,e=>`_${e.toLowerCase()}`)}function ee(e){switch(e.action){case O.length:return[t(`jsonb_array_length(${k(e.columnName)})`),e.alias];default:return e.action,[]}}function te(e){return e.map(e=>ee(e))}function ne(e){return e.map(e=>{let n=k(e.columnName),r=`json_build_object(${e.keys.map(e=>`'${e}', ${n} -> '${e}'`).join(`, `)})`,i=e.alias||e.columnName;return[t(r),i]})}function A({select:e=[],computed:t=[]}={}){let n=ne(e),r=te(t);return[...n,...r]}const j=`DESC`,M=`customFields.`,{CUSTOM_FIELDS_FILTER_SCOPE:N,CUSTOM_FIELDS_SORT_SCOPE:P}=c,re=e=>[`string`,`number`].includes(typeof e)||Array.isArray(e)?e:Object.entries(e).map(([e,t])=>({operator:_[e],value:t})),ie=(e,t={})=>{let{literalAttributes:n=[],DBFormatter:r=void 0}=t,[i,a]=e.reduce((e,t)=>{let[i,a=`ASC`]=Array.isArray(t)?t:[t],o=n?.find(e=>e.attribute===i);if(o){let t=r?r(`"${o.attribute}" ${a}`):`${o.attribute} ${a}`;e[1].push(o.literal),e[0].push([t])}else e[0].push(t);return e},[[],[]]);return[i,a]},F=e=>{let t={};return Object.entries(e).forEach(([e,n])=>{let r=E();if(t[r]=e.split(M,2)[1],Array.isArray(n))n.forEach(e=>{let n=E();t[n]=typeof e==`string`?e:e.value});else if(typeof n==`string`||typeof n==`number`){let e=E();t[e]=n}else if(n?.operator){let e=E();t[e]=n.value}}),t},ae=e=>{let t={};return e.forEach(e=>{if(e.startsWith(M)){let n=E();t[n]=e.split(M,2)[1]}else if(e.substring(1).startsWith(M)){let n=E();t[n]=e.substring(1).split(M,2)[1]}}),t},oe=(e,t)=>({...ae(e),...F(t)}),se=({order:e,associationModels:t=[],replacementsMap:n={}})=>{let r=[],i=new Map;return e.forEach(e=>{if([e,e.substring(1)].some(e=>e.startsWith(M))){i.has(P)||i.set(P,{});let t=e.split(M,2)[1];i.get(P)[t]=C(e)?j:`ASC`;return}let n=[S(e,t)],a=C(e);x(a?e.split(`-`,2)[1]:e,t)&&n.push(T(e)),a&&n.push(j),r.push(n)}),{formattedOrders:r,replacementsMap:n,orderScopes:Array.from(i.entries()).map(([e,t])=>t?{method:[e,{replacementsMap:n,scopeValue:t}]}:e)}},ce=e=>e||1,le=e=>e||20,ue=e=>{let n=e=>e.replace(/(?!^)[A-Z]/g,e=>`_${e.toLowerCase()}`);if(e.action===`timezoneConvert`){let r=e.additionalPayload?.timezoneColumn;if(!r)throw Error(`timezoneColumn required for timezoneConvert`);return t(`(${n(e.columnName)} AT TIME ZONE ${n(r)})`)}throw Error(`Unknown action: ${e.action}`)},de=(t,r=[])=>{if(!t||!r.length)return t;let i=[],a={};for(let[e,o]of Object.entries(t)){let t=r.find(t=>t.alias===e);if(t){let e=ue(t);i.push(n(e,o))}else a[e]=o}return i.length===0?a:Object.keys(a).length===0?i.length===1?i[0]:{[e.and]:i}:{[e.and]:[...i,a]}},I=(e,t={})=>{let n=e.map(e=>{let n=t[typeof e==`string`?e:e.association||e.model],r=typeof e!=`string`&&e.where;return typeof e!=`string`&&e.customWhereClauses&&e.where&&(r=de(e.where,e.customWhereClauses)),{...typeof e!=`string`&&e,association:n,required:typeof e==`string`||e.required!==!1,...r&&{where:r},...typeof e!=`string`&&e.include&&{include:I(e.include,n?.target?.associations)}}});return n=n.map(({model:e,customWhereClauses:t,...n})=>n),n},L=e=>{if(e==null)return e;if(Array.isArray(e))return e.map(e=>L(e));if(typeof e==`object`&&e.constructor===Object){let t={};for(let[n,r]of Object.entries(e)){let e=v[n]??n;t[e]=L(r)}return t}return e},fe=({query:e,associationModels:t,replacementsMap:n,additionalAllowedAttributes:r=[],useOpSymbols:i=!1})=>{let a={},o={},s=new Map;Object.entries(e).forEach(([e,n])=>{if(e.startsWith(M)){s.has(N)||s.set(N,{});let t=e.split(M,2)[1];s.get(N)[t]=re(n);return}if(r.includes(e)){o[e]=n;return}let i=x(e,t)?b(e):e;a[i]=n});let c=Array.from(s.entries()).map(([e,t])=>t?{method:[e,{replacementsMap:n,scopeValue:t}]}:e);return{formattedQuery:i?L(a):a,externalQueryValues:o,formattedScopes:c}},pe=({searchTerm:t,attributesToSend:n,rawAttributes:r,useOpSymbols:i=!1})=>({[i?e.and:`$and`]:t.split(` `).map(t=>({[i?e.or:`$or`]:n.filter(e=>r[e].type.key===`STRING`).map(n=>({[n]:{[i?e.iLike:`$iLike`]:`%${t}%`}}))}))});var me=({order:t=[],page:n=1,perPage:r=20,include:i=[],query:a={},attributes:o=null,searchTerm:s=null,jsonAttributes:c={},subQuery:l},u,d)=>{let f=oe(t,a),p=Object.keys(u?.associations||{}),{formattedOrders:m,orderScopes:h}=se({order:[...t,`id`],associationModels:p,replacementsMap:f}),[g,_]=ie(m,d),v=A(c),y=[..._,...o??[],...v],b=o?.length?y:{include:y},x=I(i,u?.associations),S=ce(n),C=le(r),w=fe({query:a,associationModels:p,replacementsMap:f,additionalAllowedAttributes:d?.additionalAllowedAttributes,useOpSymbols:d?.useOpSymbols}),{formattedScopes:T,externalQueryValues:E}=w,{formattedQuery:D}=w;if(s&&!d?.skipSearchTermFormat){let t=pe({searchTerm:s,attributesToSend:o?.length?o:Object.keys(u.rawAttributes||{}),rawAttributes:u.rawAttributes,useOpSymbols:d?.useOpSymbols}),n=!Object.keys(D).length&&!Object.getOwnPropertySymbols(D).length;D=!D||n?t:{[d?.useOpSymbols?e.and:`$and`]:[D,t]}}return{query:D,order:g,page:S,perPage:C,include:x,scopes:[...T,...h],...b&&{attributes:b},...Object.keys(E).length>0&&{externalQueryValues:E},...l!==void 0&&{subQuery:l}}};const R=e=>g.includes(e.split(`$`,2)[1]),z=(e,t=[],n=[],r=[])=>{let i=e.startsWith(`$`)&&e.endsWith(`$`)?e.slice(1,-1):e;return[...t,...n].includes(i.includes(`.`)?i.split(`.`,1)[0]:i)||r.includes(i)},he=(e,t,n,r={})=>{let i=C(e);i&&!e.startsWith(`-`)&&w(`- must be only at the beginning of the word`);let a=i?e.split(`-`,2)[1]:e,o=x(a,n),s=S(e,n),c=r?.literalAttributes?.map(e=>e.attribute)?.includes(a);!o&&s.includes(`.`)&&([s]=s.split(`.`,1)),t.includes(s)||o||c||w(`${e} is invalid. isLiteralAttribute: ${c}`)},ge=(e,t)=>{t.includes(e)||w(`${e} is invalid`)},B=(e,t,n=[],r={})=>{e.forEach(e=>he(e,t,n,r))},V=(e,t)=>{e.forEach(e=>ge(e,t))},_e=(e,t)=>{V([...e.select?.map(e=>e.columnName)??[],...e.computed?.map(e=>e.columnName)??[]],t)},ve=(e,t)=>{let n=Array.isArray(e)?e:Object.keys(e);if(!n?.length)return;let r=n.find(e=>!t?.enrichmentAttributes?.includes(e));r&&w(`enrichment attribute ${r} is invalid`)},H=(e,t,n=[],r=[])=>{Object.entries(e).forEach(([e,i])=>{Array.isArray(i)?i[0]&&typeof i[0]==`object`&&i.map(e=>H(e,t,n,r)):R(e)||z(e,t,n,r)?i&&typeof i==`object`&&H(i,t,[],r):w(`invalid key: ${e}`)})},ye=({page:e,perPage:t})=>{e<1&&w(`Page must be greater than 0`),(t>100||t<1)&&w(`PerPage must be between 1 to 100`)},be=(e,t)=>{let n=Object.keys(t);e.forEach(e=>{z(e.association||e.model,n);let r=t[e.association||e.model]?.target;r||w(`model not found in associations`);let{rawAttributes:i}=r,a=Object.keys(i),o=e.customWhereClauses?.map(e=>e.alias)??[];e.where&&H(e.where,a,[],o),e.order&&B(e.order,a),e.attributes&&V(e.attributes,a),[null,void 0,!0,!1].includes(e.required)||w(`include.required must be a boolean`)})},xe=e=>{e===void 0||typeof e==`boolean`||w(`subQuery must be a boolean or undefined`)},U=({query:e={},order:t=[],attributes:n=[],include:r=[],page:i=1,perPage:a=20,enrichments:o=[],group:s=[],jsonAttributes:c={},subQuery:l},u,d={})=>{let f=Object.keys(u.rawAttributes),p=Object.keys(u?.associations||{});return!n||n.length===0?n=f:V(n,f),B(t,f,p,d),H(e,f,p,d.additionalAllowedAttributes),ve(o,d),_e(c,f),Array.isArray(s)||w(`group must be an array`),r.length&&typeof r==`object`?be(r,u?.associations):r&&typeof r!=`object`&&w(`include must be an array`),ye({page:i,perPage:a}),xe(l),!0},{object:W,string:G,number:K,any:Se,array:q,alternatives:Ce}=s.types(),we=r(),Te=W.keys({query:W,attributes:q.items(G),order:q.items(G),page:K,perPage:K,include:q.items(Se),searchTerm:G,group:q.items(G),enrichments:Ce.try(q.items(G),W.pattern(G,{exclude:q.items(G)})),jsonAttributes:s.object({select:s.array().items(s.object({columnName:s.string().required(),keys:s.array().items(s.string().required()).required(),alias:s.string().optional()})).default([]),computed:s.array().items(s.object({columnName:s.string().required(),action:s.string().valid(...Object.values(O)).required(),alias:s.string().required()})).default([])}).default({}),subQuery:s.boolean()}),J=(e,t,n={})=>{let{query:r,attributes:a,order:o,page:s,perPage:c,include:l,group:u,enrichments:d,jsonAttributes:f,subQuery:p}=t,m=Te.validate(t);if(m.error)throw new i([m.error]);U({query:r,attributes:a,order:o,page:s,perPage:c,include:l,enrichments:d,group:u,jsonAttributes:f,subQuery:p},e,n)},Ee=(e,t={},n=`body`)=>(r,i,a)=>{try{J(e,r[n],t),a()}catch(e){let{query:a,attributes:s,order:c}=r[n];o(e,i,{logger:t.logger??we,message:`error in query middleware`,payload:{error:e,query:a,attributes:s,order:c}})}},Y=(e,t,n={})=>{let{order:r,page:i,perPage:a,include:o,query:s,attributes:c,searchTerm:l,jsonAttributes:u,subQuery:d}=t,{query:f,externalQueryValues:p,order:m,page:h,perPage:g,include:_,scopes:v,attributes:y,subQuery:b}=me({query:s,order:r,page:i,perPage:a,include:o,attributes:c,searchTerm:l,jsonAttributes:u,subQuery:d},e,n);t.query=f,t.externalQueryValues=p,t.order=m,t.attributes=y,t.page=h,t.perPage=g,t.include=_,t.scopes=v,b!==void 0&&(t.subQuery=b),n.includeRawPayload&&(t.rawPayload={order:r,page:i,perPage:a,include:o,query:s,attributes:c,searchTerm:l,subQuery:d})},De=(e,t={},n=`body`)=>(r,i,a)=>{Y(e,r[n],t),a()},Oe=({model:e,logger:t,validationOptions:n,formatOptions:r,modelName:i=e.constructor?.name,additionalScopes:s=[],modifyQueryValues:c,onRowsRetrieved:l})=>async(u,d)=>{try{J(e,u.body,{...n,logger:t})}catch(e){o(e,d,{logger:t,message:`error in query endpoint`,payload:D(u.body,[`query`,`order`,`attributes`])});return}try{Y(e,u.body,r);let n=Object.assign(D(u.body,[`query`,`externalQueryValues`,`order`,`attributes`,`page`,`perPage`,`include`,`scopes`,`enrichments`,`subQuery`]),{distinct:!0});t.info(`querying ${i}`,{queryValues:n});let a=c?.(n)??n,{scopes:o=[],query:f,perPage:p,page:m,enrichments:h,externalQueryValues:g,subQuery:_,...v}=a,y=await e.scope([...s,...o]).findAndCountAll({where:f,limit:p,offset:(m-1)*p,...v,..._!==void 0&&{subQuery:_}});if(!y.rows.length||!l){d.json(y);return}let b=await l(y,a);d.json(b)}catch(e){o(new a(e),d,{logger:t,message:`Error while querying ${i}`,payload:{query:u.body}})}};var X=class e extends Error{constructor(e,t=!1){super(e),this.name=`BulkerError`,this.retryable=t,Object.setPrototypeOf(this,new.target.prototype)}static isBulkerError(t){return t instanceof e}static wrap(t,n,r=!1){return t instanceof e?new e(n??t.message,r||t.retryable):new e(n?`${n}: ${t instanceof Error?t.message:String(t)}`:String(t),r)}static retryable(t){return new e(t,!0)}static nonRetryable(t){return new e(t,!1)}};const ke=e=>e.length===0?`No keys provided`:e.length===1?`Key "${e[0]}" is required`:`Exactly one of [${e.join(`, `)}] must be provided`;function Z(e,t){return m.object(e).refine(e=>t.filter(t=>e[t]!==void 0&&e[t]!==null).length===1,{message:ke(t)})}const Q=m.string().uuid();function Ae(e){return Z(Object.fromEntries(e.map(e=>[e,m.union([Q,m.array(Q)]).optional()])),e)}const je=[`businessModelId`,`fleetId`,`demandSourceId`,`contextId`,`userId`,`businessAccountId`,`activeBusinessModelId`];var Me=class{constructor(e,t){if(this.bulker=e,this.opts=t,this.rabbitQueueName=null,this.rabbit=null,this.bulkHandler=async(e,t,n)=>{try{let{query:n,payload:r,preview:i}=e.body??{},a=r;if(r&&this.payloadSchema)try{a=this.payloadSchema.parse(r)}catch(e){if(e instanceof m.ZodError)return t.status(400).json({error:`invalid_payload`,details:e.issues.map(e=>`${e.path.join(`.`)}: ${e.message}`).join(`, `)});throw e}try{J(this.model,n,{logger:this.bulker.logger})}catch(e){return t.status(400).json({error:`invalid_query`,details:e.message||e})}let o=this.identityScopeSchema.safeParse(n?.query||{});if(!o.success)return t.status(400).json({error:`invalid_identity_scope`,details:o.error.issues.map(e=>`${e.path.join(`.`)}: ${e.message}`).join(`, `)});Y(this.model,n,{includeRawPayload:!0});let{query:s,...c}=Object.assign(D(n,[`query`,`externalQueryValues`,`order`,`include`,`scopes`,`enrichments`]),{distinct:!0}),l=[];if(this.opts.additionalIdsHook){let{rawPayload:e}=n;try{l=await this.opts.additionalIdsHook({rawPayload:e,payload:a}),this.bulker.logger.info(`additionalIdsHook returned ${l.length} IDs for action ${this.action}`)}catch(e){return this.bulker.logger.error(`Error in additionalIdsHook for action ${this.action}: ${e.message||e}`,{err:e}),t.status(500).json({error:`additional_ids_hook_error`,details:e.message||e})}}let d=[];s&&typeof s==`object`&&Object.keys(s).length>0&&d.push(s),Array.isArray(l)&&l.length>0&&d.push({id:{$in:l}});let f;if(d.length===0)return t.status(400).json({error:`no_query`,details:`No valid query provided to select records`});if(d.length===1){let[e]=d;f=e}else f={$or:d};this.bulker.logger.info(`Constructed final where clause for action ${this.action}`,{where:f});let p={where:f,...c},h=await this.model.scope(this.modelScopes).count({...p,col:this.idField});if(i)return t.json({estimatedCount:h});let g=u();return await this.bulker.jobManager.initJob(g,{status:`queued`,total:h,action:this.action}),this.bulker.emitEvent(`job:created`,{jobId:g,action:this.action,total:h}),setImmediate(()=>{this.rabbitScanAndEnqueue(g,p,a).catch(e=>this.bulker.logger.error(e))}),t.status(202).json({jobId:g,estimatedCount:h})}catch(e){return this.bulker.logger.error(`Error in bulkHandler for action ${this.action}: ${e.message||e}`,{err:e}),n(e)}},this.action=t.action,this.model=t.model,this.modelScopes=t.modelScopes??[],this.consumer=t.consumer,this.rabbit=e.rabbit,this.rabbitQueueName=t.rabbitQueueName??`bulk-${this.action}-queue`,this.consumerOptions={enableRabbitTrace:!0,...this.opts.consumerOptions},this.payloadSchema=t.payloadSchema,!/^[a-zA-Z0-9-_]+$/.test(this.action))throw Error(`BulkRoute action must be alphanumeric`);if(!this.model||typeof this.model.findAll!=`function`||typeof this.model.count!=`function`)throw Error(`BulkRoute model must be a valid Sequelize model`);if(typeof this.consumer!=`function`)throw Error(`BulkRoute consumer must be a function`);if(this.payloadSchema&&!(this.payloadSchema instanceof m.ZodType))throw Error(`BulkRoute payloadSchema must be a Zod schema`);if(this.modelScopes&&!Array.isArray(this.modelScopes))throw Error(`BulkRoute scopes must be an array of strings`);if(this.opts.additionalIdsHook&&typeof this.opts.additionalIdsHook!=`function`)throw Error(`BulkRoute additionalIdsHook must be a function`);this.idField=t.idField??e.defaults.idField,this.pageSize=t.pageSize??e.defaults.pageSize,this.consumerBatchSize=t.consumerBatchSize??e.defaults.consumerBatchSize;let n=this.model.rawAttributes||{},r=(t.identityScopes&&t.identityScopes.length>0?t.identityScopes:je).filter(e=>!!n[e]);r.length===0&&this.bulker.logger.warn(`BulkRoute for action ${this.action} has no valid identityScopes configured - all records will be accessible`),this.bulker.logger.info(`BulkRoute for action ${this.action} using idField ${this.idField}, pageSize ${this.pageSize},
|
|
2
|
+
consumerBatchSize ${this.consumerBatchSize}, identityScopes: ${r.join(`, `)}`),this.identityScopeSchema=Ae(r),this.startRabbitWorker().catch(e=>{this.bulker.logger.error(`Failed to start RabbitMQ worker for queue ${this.rabbitQueueName}: ${e.message||e}`)})}async getUserJobs(e,t=20){return this.bulker.jobManager.getUserJobs(e,t)}batchIds(e){let t=[];for(let n=0;n<e.length;n+=this.consumerBatchSize)t.push(e.slice(n,n+this.consumerBatchSize));return t}async rabbitScanAndEnqueue(t,n,r){if(!this.rabbit)throw Error(`RabbitMQ not configured in Bulker`);let i=Date.now().toString();if(this.bulker.getUserId){let e=this.bulker.getUserId();e&&await this.bulker.jobManager.addUserJob(e,t)}await this.bulker.jobManager.setJobFields(t,{status:`running`,startTime:i}),this.bulker.emitEvent(`job:started`,{jobId:t,action:this.action});let a=null,o=0,s=0;for(;;){let i=await this.bulker.jobManager.getJobField(t,`status`);if(!i||i===`canceled`){this.bulker.logger.info(`Job ${t} was canceled, stopping scan and enqueue`);break}let c=a?{[e.and]:[n.where,{[this.idField]:{[e.gt]:a}}]}:n.where,l=await this.model.scope(this.modelScopes).findAll({where:c,...D(n,[`include`,`order`,`scopes`]),attributes:[this.idField],order:[[this.idField,`ASC`]],limit:this.pageSize,raw:!0,subQuery:!1});if(l.length===0)break;let u,d=l.map(e=>e[this.idField]);this.consumerBatchSize===1&&d.length===1&&(u=d[0]);let f=this.batchIds(d).map(e=>({jobId:t,ids:e,payload:r,id:u}));this.bulker.logger.info(`Enqueuing ${f.length} batched messages (${d.length} total IDs) to RabbitMQ queue ${this.rabbitQueueName}`);let p=(await Promise.allSettled(f.map(e=>this.rabbit.sendToQueue(this.rabbitQueueName,e)))).filter(e=>e.status===`rejected`);if(p.length>0)throw this.bulker.logger.error(`Failed to enqueue ${p.length} messages to RabbitMQ`,{rejected:p}),Error(`Failed to enqueue ${p.length} messages to RabbitMQ`);o+=l.length,await this.bulker.jobManager.incrJobField(t,`queued`,l.length),a=l[l.length-1][this.idField],s+=1,this.bulker.emitEvent(`scan:page`,{jobId:t,action:this.action,pageNumber:s,itemsInPage:l.length});let m=await this.bulker.jobManager.getJob(t);m&&this.bulker.emitEvent(`job:queued`,{jobId:t,action:this.action,queued:o,total:m.total})}o===0&&await this.bulker.jobManager.completeEmptyJob(t)}async startRabbitWorker(){return this.bulker.logger.info(`Starting RabbitMQ consumer for queue ${this.rabbitQueueName}`),this.bulker.emitEvent(`worker:started`,{action:this.action,queueName:this.rabbitQueueName}),this.rabbit?.consume(this.rabbitQueueName,async(e,t,n)=>{if(!e)return;let{jobId:r,ids:i,payload:a}=e.content,o=!1,s=async()=>{if(!o){o=!0,await Promise.all([this.bulker.jobManager.ack(r,i.length),t()]);for(let e of i)this.bulker.emitEvent(`item:processed`,{jobId:r,action:this.action,id:e})}},c=async(e,t)=>{if(o)return;o=!0;let a=t;a===void 0&&(a=X.isBulkerError(e)?e.retryable:!1),await Promise.all([this.bulker.jobManager.nack(r,e?e.message||String(e):`nacked`,{ids:i},i.length),n(null,{skipRetry:!a})]);for(let t of i)a?this.bulker.emitEvent(`item:retrying`,{jobId:r,action:this.action,id:t}):this.bulker.emitEvent(`item:failed`,{jobId:r,action:this.action,id:t,error:e?e.message||String(e):`nacked`})},l=async e=>{if(o)return;o=!0;let n=e.succeeded??[],i=e.failed??[];await Promise.all([this.bulker.jobManager.partialAck(r,n.length,i),t()]);for(let e of n)this.bulker.emitEvent(`item:processed`,{jobId:r,action:this.action,id:e});for(let e of i)this.bulker.emitEvent(`item:failed`,{jobId:r,action:this.action,id:e.id,error:e.error||`Item failed`})},u=await this.bulker.jobManager.getJobField(r,`status`);if(!u||u===`canceled`){await this.bulker.jobManager.incrJobField(r,`processed`,i.length);return}try{this.bulker.logger.info(`Started processing job ${r} action ${this.action} ids amount: ${i.length}`);for(let e of i)this.bulker.emitEvent(`item:processing`,{jobId:r,action:this.action,id:e});await this.consumer({jobId:r,ids:i,id:this.consumerBatchSize===1?i[0]:void 0,payload:a},s,c,l),await s()}catch(e){this.bulker.logger.error(`Error processing job ${r} action ${this.action}: ${e.message||e}`,{err:e}),this.bulker.emitEvent(`worker:error`,{action:this.action,error:e.message||String(e)}),await c(e,!1)}},this.consumerOptions)}},Ne=class{constructor(e,t,n=`/bulk-actions`){this.routes=[],this.actionsToRouteMap=new Map,this.bulkHandler=async(e,t,n)=>{try{let{action:r}=e.body??{};if(!r)return t.status(400).json({error:`missing_action`,message:`Request body must include an "action" field`});let i=this.actionsToRouteMap.get(r);return i?i.bulkHandler(e,t,n):t.status(404).json({error:`unknown_action`,message:`Action "${r}" is not registered`,availableActions:Array.from(this.actionsToRouteMap.keys())})}catch(e){return this.bulker.logger.error(`Error in BulkRouter bulkHandler: ${e.message||e}`,{err:e}),n(e),null}},this.getJobHandler=async(e,t,n)=>{try{if(!e.params.id)return t.status(400).json({error:`missing_id`});let n=await this.bulker.jobManager.getJob(e.params.id);return n?t.json(n):t.status(404).json({error:`not_found`})}catch(e){return n(e)}},this.cancelJobHandler=async(e,t,n)=>{try{return e.params.id?await this.bulker.jobManager.cancelJob(e.params.id)?(this.bulker.logger.info(`Job ${e.params.id} cancel requested`),t.json({ok:!0})):t.status(404).json({error:`not_found`}):t.status(400).json({error:`missing_id`})}catch(e){return n(e)}},this.getMyJobsHandler=async(e,t,n)=>{try{if(!this.bulker.getUserId)return t.status(400).json({error:`user_id_function_not_configured`,message:`Bulker instance does not have a getUserId function configured`});let e=this.bulker.getUserId();if(!e)return t.json([]);let n=(await this.bulker.jobManager.getUserJobs(e)).filter(e=>e!==null);return n.sort((e,t)=>{let n=e.createdAt?new Date(e.createdAt).getTime():0;return(t.createdAt?new Date(t.createdAt).getTime():0)-n}),t.json(n)}catch(e){return n(e)}},this.bulker=e,this.router=t??f(),this.staticRoute=n,this.registerStaticRoutes()}registerStaticRoutes(){this.router.post(this.staticRoute,this.bulkHandler),this.router.get(`/jobs/:id`,this.getJobHandler),this.router.get(`/jobs`,this.getMyJobsHandler),this.router.post(`/jobs/:id/cancel`,this.cancelJobHandler)}addAction(e,t){let n=new Me(this.bulker,{action:e,...t});if(this.bulker.logger.info(`Registering action handler: ${e}`),this.actionsToRouteMap.has(e))throw Error(`Action "${e}" is already registered`);return this.actionsToRouteMap.set(e,n),this.routes.push(n),n}getRouter(){return this.router}},$=class e{constructor(e,t){this.redis=e,this.defaults={maxJobsPerUser:t?.maxJobsPerUser??-1,jobTtlSeconds:t?.jobTtlSeconds??168*3600,errorLogLimit:t?.errorLogLimit??10}}setEventEmitter(e){this.eventEmitter=e}static jobKey(e){return`bulker:job:${e}`}static userJobsKey(e){return`bulker:user:${e}:jobs`}async initJob(t,n){let r=Date.now().toString(),i=e.jobKey(t),a=this.redis.multi();a.hSet(i,{status:n.status,total:String(n.total),queued:`0`,processed:`0`,succeeded:`0`,failed:`0`,action:n.action,errors:JSON.stringify([]),createdAt:r,updatedAt:r,startTime:r,endTime:``}),a.expire(i,this.defaults.jobTtlSeconds),await a.exec()}async getJob(t){let n=e.jobKey(t),r=await this.redis.hGetAll(n);if(!r||Object.keys(r).length===0)return null;let i=r.startTime?Number(r.startTime):null,a=r.endTime&&r.endTime!==``?Number(r.endTime):null,o=Date.now(),s={startTime:i?new Date(i).toISOString():null,endTime:a?new Date(a).toISOString():null,durationMs:i?(a||o)-i:null};return{jobId:t,status:r.status,action:r.action,total:Number(r.total??0),queued:Number(r.queued??0),processed:Number(r.processed??0),succeeded:Number(r.succeeded??0),failed:Number(r.failed??0),errors:r.errors?JSON.parse(r.errors):[],createdAt:r.createdAt?new Date(Number(r.createdAt)).toISOString():void 0,updatedAt:r.updatedAt?new Date(Number(r.updatedAt)).toISOString():void 0,duration:s}}async setJobField(t,n,r){let i=e.jobKey(t);if(!await this.redis.exists(i))return!1;let a=this.redis.multi();return a.hSet(i,n,r),a.hSet(i,`updatedAt`,Date.now().toString()),await a.exec(),!0}async setJobFields(t,n){let r=e.jobKey(t);if(!await this.redis.exists(r))return!1;let i=this.redis.multi();return i.hSet(r,n),i.hSet(r,`updatedAt`,Date.now().toString()),await i.exec(),!0}async incrJobField(t,n,r=1){let i=e.jobKey(t),a=this.redis.multi();a.hIncrBy(i,n,r),a.hSet(i,`updatedAt`,Date.now().toString());let o=await a.exec();return o?.[0]?.[0]===null?o[0][1]:0}async getJobField(t,n){let r=e.jobKey(t);return this.redis.hGet(r,n)}async cancelJob(e){return this.setJobField(e,`status`,`canceled`)}async completeEmptyJob(e){let t=Date.now().toString();await this.setJobFields(e,{status:`completed`,endTime:t})}async ack(e,t=1){await this.partialAck(e,t,[])}async nack(e,t,n,r=1){let i=[],a=e=>n.ids&&Array.isArray(n.ids)?n.ids[e]??`unknown-${e}`:`unknown-${e}`;for(let e=0;e<r;e++)i.push({id:a(e),error:t});await this.partialAck(e,0,i)}async partialAck(t,n,r){let i=e.jobKey(t);await this.redis.eval(`
|
|
3
3
|
local jobKey = KEYS[1]
|
|
4
4
|
local succeededCount = tonumber(ARGV[1]) or 0
|
|
5
5
|
local failedCount = tonumber(ARGV[2]) or 0
|
|
@@ -62,5 +62,5 @@ if processed >= total then
|
|
|
62
62
|
end
|
|
63
63
|
|
|
64
64
|
return processed
|
|
65
|
-
`,{keys:[i],arguments:[String(n),String(r.length),JSON.stringify(r),String(this.defaults.errorLogLimit),Date.now().toString()]});let a=await this.getJob(t);if(a?.status===`completed`){let e=a.duration.durationMs||0;this.eventEmitter?.emitEvent(`job:completed`,{jobId:t,action:a.action,processed:a.processed,failed:a.failed,duration:e})}}async addUserJob(t,n){let r=e.userJobsKey(t),i=this.redis.multi();i.lPush(r,n),this.defaults.maxJobsPerUser>0&&i.lTrim(r,0,this.defaults.maxJobsPerUser-1),i.expire(r,3600),await i.exec()}async removeUserJob(t,n){let r=e.userJobsKey(t);await this.redis.lRem(r,0,n)}async getUserJobs(t,n=20){let r=e.userJobsKey(t),i=await this.redis.lRange(r,0,n-1);return await Promise.all(i.map(e=>this.getJob(e)))}};const
|
|
65
|
+
`,{keys:[i],arguments:[String(n),String(r.length),JSON.stringify(r),String(this.defaults.errorLogLimit),Date.now().toString()]});let a=await this.getJob(t);if(a?.status===`completed`){let e=a.duration.durationMs||0;this.eventEmitter?.emitEvent(`job:completed`,{jobId:t,action:a.action,processed:a.processed,failed:a.failed,duration:e})}}async addUserJob(t,n){let r=e.userJobsKey(t),i=this.redis.multi();i.lPush(r,n),this.defaults.maxJobsPerUser>0&&i.lTrim(r,0,this.defaults.maxJobsPerUser-1),i.expire(r,3600),await i.exec()}async removeUserJob(t,n){let r=e.userJobsKey(t);await this.redis.lRem(r,0,n)}async getUserJobs(t,n=20){let r=e.userJobsKey(t),i=await this.redis.lRange(r,0,n-1);return await Promise.all(i.map(e=>this.getJob(e)))}};const Pe={emitEvent:()=>{}};var Fe=class extends h{constructor(e){if(super(),this.bulkRouters=[],this.sequelize=e.sequelize,this.logger=e.logger,this.rabbit=e.rabbit,this.eventsEnabled=e.emitEvents??!1,e.redis&&typeof e.redis.connect==`function`)this.redis=e.redis,this.redis.isOpen||this.redis.connect().catch(e=>{this.logger.error(`Error connecting to Redis:`,e)});else{let t=e.redis;this.redis=d({socket:{connectTimeout:6e4,host:t.host,port:t.port,reconnectStrategy(t,n){return e.logger.warn(`[BULKER Redis] Reconnecting to Redis (attempt ${t}). Cause: ${n?.message}`),Math.min(t*50,2e3)}},commandsQueueMaxLength:5e3,disableOfflineQueue:!1,password:t.password,database:t.db}),this.redis.connect().catch(e=>{this.logger.error(`Error connecting to Redis:`,e)})}this.getUserId=e.getUserId??(()=>null),this.defaults={maxJobsPerUser:e.defaults?.maxJobsPerUser??-1,pageSize:e.defaults?.pageSize??1e3,idField:e.defaults?.idField??`id`,consumerBatchSize:e.defaults?.consumerBatchSize??1,workerConcurrency:e.defaults?.workerConcurrency??16,jobTtlSeconds:e.defaults?.jobTtlSeconds??168*3600,errorLogLimit:e.defaults?.errorLogLimit??10},this.jobManager=new $(this.redis,{maxJobsPerUser:this.defaults.maxJobsPerUser,jobTtlSeconds:this.defaults.jobTtlSeconds,errorLogLimit:this.defaults.errorLogLimit}),this.jobManager.setEventEmitter(this.eventsEnabled?{emitEvent:this.emitEvent.bind(this)}:Pe)}pingRedis(){return this.redis.ping()}createBulkRouter(e,t){let n=new Ne(this,e,t);return this.bulkRouters.push(n),n}emitEvent(e,...t){this.eventsEnabled&&this.emit(e,...t)}};export{Fe as Bulker,X as BulkerError,$ as BulkerJobManager,y as formatOperators,F as generateFilterReplacements,De as queryFormatMiddleware,Oe as queryHandler,Ee as queryValidationMiddleware,U as validatePayload,p as z};
|
|
66
66
|
//# sourceMappingURL=index.js.map
|