@nikx/dory-worker 1.0.1 → 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/processor.d.ts.map +1 -1
- package/dist/processor.js +4 -2
- package/dist/processor.js.map +1 -1
- package/package.json +1 -1
- package/src/processor.ts +6 -2
package/dist/processor.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"processor.d.ts","sourceRoot":"","sources":["../src/processor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"processor.d.ts","sourceRoot":"","sources":["../src/processor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAkD7C;;;;;;;;;;GAUG;AACH,wBAAsB,aAAa,CACjC,MAAM,EAAE,YAAY,EACpB,GAAG,EAAE,GAAG,CAAC;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,GAC1B,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,CAuI5C"}
|
package/dist/processor.js
CHANGED
|
@@ -59,7 +59,9 @@ async function processRunJob(config, job) {
|
|
|
59
59
|
if (!dockerImage) {
|
|
60
60
|
throw new Error(`No docker image: API did not return one and DOCKER_IMAGE env var is not set`);
|
|
61
61
|
}
|
|
62
|
-
|
|
62
|
+
// API-returned containerCount takes precedence over the worker env var
|
|
63
|
+
const containerCount = runConfig.containerCount ?? config.containerCount;
|
|
64
|
+
const isDistributed = !!config.crawlerRedisUrl || containerCount > 1;
|
|
63
65
|
let containerIds;
|
|
64
66
|
try {
|
|
65
67
|
if (isDistributed) {
|
|
@@ -67,7 +69,7 @@ async function processRunJob(config, job) {
|
|
|
67
69
|
throw new Error("CONTAINER_COUNT > 1 but CRAWLER_REDIS_URL is not set — containers cannot share a queue");
|
|
68
70
|
}
|
|
69
71
|
containerIds = await (0, docker_1.startContainersForRun)(config, runConfig.actorConfig, runConfig.memoryLimitMb, dockerImage, config.crawlerRedisUrl, runId, // QUEUE_NAME = runId for job isolation
|
|
70
|
-
|
|
72
|
+
containerCount);
|
|
71
73
|
logger_1.logger.info(`[${runId}] ${containerIds.length} containers started (distributed mode, queue: ${runId})`);
|
|
72
74
|
}
|
|
73
75
|
else {
|
package/dist/processor.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"processor.js","sourceRoot":"","sources":["../src/processor.ts"],"names":[],"mappings":";;
|
|
1
|
+
{"version":3,"file":"processor.js","sourceRoot":"","sources":["../src/processor.ts"],"names":[],"mappings":";;AA8DA,sCA0IC;AAtMD,qCAIkB;AAClB,qCAAkC;AAiBlC,KAAK,UAAU,MAAM,CAAI,UAAkB,EAAE,IAAY;IACvD,MAAM,GAAG,GAAG,GAAG,UAAU,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC;IACvD,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACZ,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QAC9C,MAAM,IAAI,KAAK,CAAC,OAAO,IAAI,WAAW,GAAG,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC,CAAC;IAC/D,CAAC;IACD,OAAO,GAAG,CAAC,IAAI,EAAgB,CAAC;AAClC,CAAC;AAED,KAAK,UAAU,QAAQ,CACrB,UAAkB,EAClB,IAAY,EACZ,IAA6B;IAE7B,MAAM,GAAG,GAAG,GAAG,UAAU,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC;IACvD,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;QAC3B,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;QAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;KAC3B,CAAC,CAAC;IACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACZ,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QAC9C,eAAM,CAAC,IAAI,CAAC,QAAQ,IAAI,WAAW,GAAG,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC,CAAC;IAC5D,CAAC;AACH,CAAC;AAED;;;;;;;;;;GAUG;AACI,KAAK,UAAU,aAAa,CACjC,MAAoB,EACpB,GAA2B;IAE3B,MAAM,EAAE,KAAK,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC;IAE3B,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;IAEtD,eAAM,CAAC,IAAI,CACT,IAAI,KAAK,oBAAoB,GAAG,CAAC,EAAE,aAAa,GAAG,CAAC,YAAY,GAAG,CAAC,GAAG,CACxE,CAAC;IAEF,0EAA0E;IAC1E,IAAI,SAA4B,CAAC;IACjC,IAAI,CAAC;QACH,SAAS,GAAG,MAAM,MAAM,CACtB,MAAM,CAAC,UAAU,EACjB,aAAa,KAAK,SAAS,CAC5B,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,eAAM,CAAC,KAAK,CACV,IAAI,KAAK,iCAAkC,GAAa,CAAC,OAAO,EAAE,CACnE,CAAC;QACF,MAAM,GAAG,CAAC,CAAC,oBAAoB;IACjC,CAAC;IAED,0EAA0E;IAC1E,MAAM,QAAQ,CAAC,MAAM,CAAC,UAAU,EAAE,aAAa,KAAK,SAAS,EAAE;QAC7D,KAAK;QACL,MAAM,EAAE,SAAS;KAClB,CAAC,CAAC;IAEH,yEAAyE;IACzE,MAAM,WAAW,GAAG,SAAS,CAAC,WAAW,IAAI,MAAM,CAAC,WAAW,CAAC;IAChE,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CACb,6EAA6E,CAC9E,CAAC;IACJ,CAAC;IAED,uEAAuE;IACvE,MAAM,cAAc,GAAG,SAAS,CAAC,cAAc,IAAI,MAAM,CAAC,cAAc,CAAC;IACzE,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,CAAC,eAAe,IAAI,cAAc,GAAG,CAAC,CAAC;IAErE,IAAI,YAAsB,CAAC;IAC3B,IAAI,CAAC;QACH,IAAI,aAAa,EAAE,CAAC;YAClB,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;gBAC5B,MAAM,IAAI,KAAK,CACb,wFAAwF,CACzF,CAAC;YACJ,CAAC;YACD,YAAY,GAAG,MAAM,IAAA,8BAAqB,EACxC,MAAM,EACN,SAAS,CAAC,WAAW,EACrB,SAAS,CAAC,aAAa,EACvB,WAAW,EACX,MAAM,CAAC,eAAe,EACtB,KAAK,EAAE,uCAAuC;YAC9C,cAAc,CACf,CAAC;YACF,eAAM,CAAC,IAAI,CACT,IAAI,KAAK,KAAK,YAAY,CAAC,MAAM,iDAAiD,KAAK,GAAG,CAC3F,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,MAAM,EAAE,GAAG,MAAM,IAAA,uBAAc,EAC7B,MAAM,EACN,SAAS,CAAC,WAAW,EACrB,SAAS,CAAC,aAAa,EACvB,WAAW,CACZ,CAAC;YACF,YAAY,GAAG,CAAC,EAAE,CAAC,CAAC;YACpB,eAAM,CAAC,IAAI,CAAC,IAAI,KAAK,wBAAwB,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,eAAM,CAAC,KAAK,CACV,IAAI,KAAK,mCAAoC,GAAa,CAAC,OAAO,EAAE,CACrE,CAAC;QACF,MAAM,QAAQ,CAAC,MAAM,CAAC,UAAU,EAAE,aAAa,KAAK,SAAS,EAAE;YAC7D,KAAK;YACL,MAAM,EAAE,QAAQ;YAChB,KAAK,EAAE,iCAAkC,GAAa,CAAC,OAAO,EAAE;SACjE,CAAC,CAAC;QACH,MAAM,GAAG,CAAC;IACZ,CAAC;IAED,2EAA2E;IAC3E,IAAI,SAAqD,CAAC;IAC1D,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;QACd,SAAS,GAAG,WAAW,CAAC,KAAK,IAAI,EAAE;YACjC,IAAI,CAAC;gBACH,MAAM,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,KAAM,EAAE,OAAO,CAAC,CAAC;YAC5C,CAAC;YAAC,OAAO,MAAM,EAAE,CAAC;gBAChB,eAAM,CAAC,IAAI,CACT,IAAI,KAAK,4BAA6B,MAAgB,CAAC,OAAO,EAAE,CACjE,CAAC;YACJ,CAAC;QACH,CAAC,EAAE,OAAO,CAAC,CAAC;IACd,CAAC;IAED,wEAAwE;IACxE,IAAI,QAAQ,GAAG,CAAC,CAAC,CAAC;IAClB,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,GAAG,CACjC,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CACtB,IAAA,yBAAgB,EAAC,EAAE,EAAE,SAAS,CAAC,gBAAgB,CAAC,CACjD,CACF,CAAC;QACF,oEAAoE;QACpE,QAAQ,GAAG,SAAS,CAAC,MAAM,CACzB,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,EAC5C,CAAC,CACF,CAAC;QACF,eAAM,CAAC,IAAI,CACT,IAAI,KAAK,SAAS,YAAY,CAAC,MAAM,kCAAkC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAC/F,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,IAAI,SAAS;YAAE,aAAa,CAAC,SAAS,CAAC,CAAC;IAC1C,CAAC;IAED,yEAAyE;IACzE,yEAAyE;IACzE,oEAAoE;IACpE,wEAAwE;IACxE,8DAA8D;IAC9D,MAAM,WAAW,GAAG,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC5D,MAAM,QAAQ,CAAC,MAAM,CAAC,UAAU,EAAE,aAAa,KAAK,SAAS,EAAE;QAC7D,KAAK;QACL,MAAM,EAAE,WAAW;QACnB,QAAQ;QACR,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,GAAG,CAAC,QAAQ,KAAK,CAAC,IAAI;YACpB,KAAK,EAAE,8BAA8B,QAAQ,iCAAiC;SAC/E,CAAC;KACH,CAAC,CAAC;IAEH,eAAM,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,GAAG,CAAC,EAAE,mBAAmB,WAAW,EAAE,CAAC,CAAC;IACtE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;AACxC,CAAC"}
|
package/package.json
CHANGED
package/src/processor.ts
CHANGED
|
@@ -18,6 +18,8 @@ interface RunConfigResponse {
|
|
|
18
18
|
memoryLimitMb: number;
|
|
19
19
|
actorTimeoutSecs: number;
|
|
20
20
|
dockerImage: string;
|
|
21
|
+
/** Per-actor parallelism set in dory-api. Falls back to CONTAINER_COUNT env var (default 1). */
|
|
22
|
+
containerCount?: number;
|
|
21
23
|
}
|
|
22
24
|
|
|
23
25
|
async function apiGet<T>(apiBaseUrl: string, path: string): Promise<T> {
|
|
@@ -98,7 +100,9 @@ export async function processRunJob(
|
|
|
98
100
|
);
|
|
99
101
|
}
|
|
100
102
|
|
|
101
|
-
|
|
103
|
+
// API-returned containerCount takes precedence over the worker env var
|
|
104
|
+
const containerCount = runConfig.containerCount ?? config.containerCount;
|
|
105
|
+
const isDistributed = !!config.crawlerRedisUrl || containerCount > 1;
|
|
102
106
|
|
|
103
107
|
let containerIds: string[];
|
|
104
108
|
try {
|
|
@@ -115,7 +119,7 @@ export async function processRunJob(
|
|
|
115
119
|
dockerImage,
|
|
116
120
|
config.crawlerRedisUrl,
|
|
117
121
|
runId, // QUEUE_NAME = runId for job isolation
|
|
118
|
-
|
|
122
|
+
containerCount,
|
|
119
123
|
);
|
|
120
124
|
logger.info(
|
|
121
125
|
`[${runId}] ${containerIds.length} containers started (distributed mode, queue: ${runId})`,
|