@lincoln504/pi-research 1.3.1 → 1.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.mjs +167 -118
- package/dist/thread-worker.mjs +29 -2
- package/docs/CHANGELOG.md +57 -0
- package/package.json +4 -4
- package/skills/pi-research/SKILL.md +1 -1
- package/src/cleanup/research-cleanup.ts +3 -35
- package/src/cli.ts +11 -3
- package/src/config.ts +28 -3
- package/src/core/interfaces/state-manager-interfaces.ts +0 -1
- package/src/core/llm/agentic-repair.ts +5 -1
- package/src/core/llm/model-registry-factory.ts +17 -7
- package/src/core/pi-version.ts +15 -2
- package/src/core/service-registry.ts +104 -36
- package/src/infrastructure/browser/thread-worker-browser.ts +53 -4
- package/src/infrastructure/browser/thread-worker.mjs +29 -2
- package/src/infrastructure/browser/thread-worker.ts +8 -0
- package/src/infrastructure/knowledge-store-service.ts +78 -32
- package/src/infrastructure/state/state-browser-api.ts +0 -22
- package/src/infrastructure/state/state-browser-manager.ts +0 -21
- package/src/infrastructure/state/state-manager-service.ts +0 -7
- package/src/infrastructure/state/state-manager.ts +0 -11
- package/src/infrastructure/state/state-path-configuration.ts +7 -3
- package/src/knowledge/store.ts +33 -4
- package/src/orchestration/researcher-executor.ts +15 -5
- package/src/tools/research-knowledge-search.ts +6 -2
- package/src/tools/research-tool-definition.ts +0 -2
- package/src/types/index.ts +0 -2
- package/src/types/research-panel-types.ts +0 -2
- package/src/utils/log-utils.ts +13 -1
- package/src/utils/url-utils.ts +15 -7
- package/src/web-research/scraper-utils.ts +9 -5
- package/src/web-research/web-scraper.ts +29 -4
|
@@ -3,7 +3,6 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Handles cleanup operations for research sessions:
|
|
5
5
|
* - Terminal input draining to prevent protocol response leaks
|
|
6
|
-
* - Wave animation timer cleanup
|
|
7
6
|
* - Session and panel cleanup
|
|
8
7
|
* - Shared links cleanup
|
|
9
8
|
*/
|
|
@@ -17,7 +16,6 @@ import {
|
|
|
17
16
|
} from '../orchestration/session-state.ts';
|
|
18
17
|
import { cleanupSharedLinks } from '../utils/shared-links.ts';
|
|
19
18
|
import type { CleanupContext } from '../types/index.ts';
|
|
20
|
-
import type { ResearchPanelState } from '../types/research-panel-types.ts';
|
|
21
19
|
|
|
22
20
|
export type { CleanupContext };
|
|
23
21
|
|
|
@@ -38,16 +36,13 @@ export function createCleanupFunction(
|
|
|
38
36
|
researchId,
|
|
39
37
|
piSessionId,
|
|
40
38
|
masterWidgetId,
|
|
41
|
-
panelState,
|
|
42
|
-
waveTimer,
|
|
43
39
|
unsubOrder,
|
|
44
40
|
} = cleanupCtx;
|
|
45
|
-
|
|
41
|
+
|
|
46
42
|
const { ctx } = deps;
|
|
47
|
-
|
|
43
|
+
|
|
48
44
|
let cleanupCalled = false;
|
|
49
|
-
|
|
50
|
-
|
|
45
|
+
|
|
51
46
|
// Use reference objects to allow updating after creation
|
|
52
47
|
const unsubOrderContainer = cleanupCtx.unsubOrderRef || { value: unsubOrder };
|
|
53
48
|
|
|
@@ -60,14 +55,6 @@ export function createCleanupFunction(
|
|
|
60
55
|
if (cleanupCalled) return;
|
|
61
56
|
cleanupCalled = true;
|
|
62
57
|
|
|
63
|
-
// Clear wave animation timer
|
|
64
|
-
if (waveTimerRef) {
|
|
65
|
-
clearInterval(waveTimerRef);
|
|
66
|
-
waveTimerRef = null;
|
|
67
|
-
panelState.waveFrame = undefined;
|
|
68
|
-
panelState.waveColors = undefined;
|
|
69
|
-
}
|
|
70
|
-
|
|
71
58
|
if (unsubOrderContainer.value) {
|
|
72
59
|
unsubOrderContainer.value();
|
|
73
60
|
unsubOrderContainer.value = null;
|
|
@@ -119,13 +106,6 @@ export function createCleanupFunction(
|
|
|
119
106
|
};
|
|
120
107
|
}
|
|
121
108
|
|
|
122
|
-
/**
|
|
123
|
-
* Update the wave timer reference in the cleanup context
|
|
124
|
-
*/
|
|
125
|
-
export function updateWaveTimer(cleanupCtx: CleanupContext, timer: NodeJS.Timeout | null): void {
|
|
126
|
-
(cleanupCtx as CleanupContext & { waveTimer?: NodeJS.Timeout | null }).waveTimer = timer;
|
|
127
|
-
}
|
|
128
|
-
|
|
129
109
|
/**
|
|
130
110
|
* Update the unsubOrder reference in the cleanup context
|
|
131
111
|
*/
|
|
@@ -133,16 +113,4 @@ export function updateUnsubOrder(cleanupCtx: CleanupContext, unsub: (() => void)
|
|
|
133
113
|
if (cleanupCtx.unsubOrderRef) {
|
|
134
114
|
cleanupCtx.unsubOrderRef.value = unsub;
|
|
135
115
|
}
|
|
136
|
-
}
|
|
137
|
-
|
|
138
|
-
/**
|
|
139
|
-
* Stop and clear wave animation
|
|
140
|
-
*/
|
|
141
|
-
export function stopWaveAnimation(panelState: ResearchPanelState): void {
|
|
142
|
-
if (panelState.waveTimer) {
|
|
143
|
-
clearInterval(panelState.waveTimer);
|
|
144
|
-
panelState.waveTimer = null;
|
|
145
|
-
}
|
|
146
|
-
panelState.waveFrame = undefined;
|
|
147
|
-
panelState.waveColors = undefined;
|
|
148
116
|
}
|
package/src/cli.ts
CHANGED
|
@@ -41,7 +41,7 @@ import { RESEARCH_STOPPED_ERROR_CODE } from './orchestration/session-state.ts';
|
|
|
41
41
|
import { validateAndSanitizeQuery } from './utils/input-validation.ts';
|
|
42
42
|
import { validateInitialLink, MAX_INITIAL_LINKS } from './utils/url-utils.ts';
|
|
43
43
|
import { exportResearchReport, appendExportMessage } from './utils/research-export.ts';
|
|
44
|
-
import { getConfig, getGlobalConfigDir, getGlobalEnvFilePath, getInterfaceEnvFilePath, saveConfig, resetConfig, describeKnowledgeStoreMode, isProjectScopedKey, parseDotEnv } from './config.ts';
|
|
44
|
+
import { getConfig, getGlobalConfigDir, getGlobalEnvFilePath, getInterfaceEnvFilePath, getStateDir, saveConfig, resetConfig, describeKnowledgeStoreMode, isProjectScopedKey, parseDotEnv } from './config.ts';
|
|
45
45
|
import { getAgentDir } from '@earendil-works/pi-coding-agent';
|
|
46
46
|
import { buildModelRegistry, safeGetAvailable } from './core/llm/model-registry-factory.ts';
|
|
47
47
|
|
|
@@ -327,7 +327,7 @@ function resolvedConfigPaths(): ResolvedConfigPaths {
|
|
|
327
327
|
piIfaceEnv: getInterfaceEnvFilePath('pi'),
|
|
328
328
|
piAuth: path.join(agentDir, 'auth.json'),
|
|
329
329
|
piModels: path.join(agentDir, 'models.json'),
|
|
330
|
-
piState:
|
|
330
|
+
piState: getStateDir(),
|
|
331
331
|
};
|
|
332
332
|
}
|
|
333
333
|
|
|
@@ -695,6 +695,13 @@ export async function cmdKnowledge(queries: string[], json?: boolean): Promise<n
|
|
|
695
695
|
|
|
696
696
|
let result: KnowledgeSearchResult;
|
|
697
697
|
let exit: number = EXIT.OK;
|
|
698
|
+
// Same rationale as cmdResearch: without wiring a signal in, Ctrl-C only tears
|
|
699
|
+
// down shared SDK services (safeShutdown, fired from onSignal) while this
|
|
700
|
+
// search keeps running unsignaled underneath it — a race that can either
|
|
701
|
+
// surface as a confusing mid-teardown failure or, worse, let the search
|
|
702
|
+
// resolve normally and report ok:true/exit 0 for an interrupted run.
|
|
703
|
+
const abortController = new AbortController();
|
|
704
|
+
activeResearchAbortController = abortController;
|
|
698
705
|
try {
|
|
699
706
|
await initResearchSDK({
|
|
700
707
|
model: det.model,
|
|
@@ -704,10 +711,11 @@ export async function cmdKnowledge(queries: string[], json?: boolean): Promise<n
|
|
|
704
711
|
ignoreGlobalConfig: true,
|
|
705
712
|
});
|
|
706
713
|
toStderr(`[pi-research] searching knowledge store (${queries.length} query/${queries.length === 1 ? '' : 's'})…\n`);
|
|
707
|
-
result = await searchKnowledge(queries);
|
|
714
|
+
result = await searchKnowledge(queries, abortController.signal);
|
|
708
715
|
} catch (err) {
|
|
709
716
|
return reportError(err, 'knowledge search', json);
|
|
710
717
|
} finally {
|
|
718
|
+
if (activeResearchAbortController === abortController) activeResearchAbortController = null;
|
|
711
719
|
await safeShutdown();
|
|
712
720
|
}
|
|
713
721
|
|
package/src/config.ts
CHANGED
|
@@ -247,6 +247,33 @@ const USER_MIGRATION_KEYS = [
|
|
|
247
247
|
// Centralized Project Settings Storage
|
|
248
248
|
// ============================================================================
|
|
249
249
|
|
|
250
|
+
/**
|
|
251
|
+
* Returns the state directory: PI_RESEARCH_STATE_DIR if set and absolute,
|
|
252
|
+
* otherwise ~/.pi/research/state.
|
|
253
|
+
*
|
|
254
|
+
* A relative override is deliberately ignored (falls back to the default)
|
|
255
|
+
* rather than resolved against process.cwd(): this directory backs
|
|
256
|
+
* cross-process coordination (browser-server/embedding-server leader
|
|
257
|
+
* election, the GPU lock, the run semaphore) that requires every process —
|
|
258
|
+
* the CLI invoked from an arbitrary cwd, the pi extension host, cluster
|
|
259
|
+
* worker processes — to agree on the exact same path. Resolving relative to
|
|
260
|
+
* whichever cwd a given process happens to be launched from would silently
|
|
261
|
+
* fragment that coordination into several independent, non-communicating
|
|
262
|
+
* state directories. scripts/cleanup.cjs already treats a relative override
|
|
263
|
+
* the same way (falls back to the default rather than guessing a cwd) —
|
|
264
|
+
* this keeps every reader in agreement with what deletion tooling assumes.
|
|
265
|
+
*/
|
|
266
|
+
export function getStateDir(): string {
|
|
267
|
+
const override = process.env['PI_RESEARCH_STATE_DIR'];
|
|
268
|
+
if (override) {
|
|
269
|
+
if (path.isAbsolute(override)) return override;
|
|
270
|
+
logger.warn(
|
|
271
|
+
`[config] PI_RESEARCH_STATE_DIR must be an absolute path — ignoring relative value "${override}" and using the default state directory.`,
|
|
272
|
+
);
|
|
273
|
+
}
|
|
274
|
+
return path.join(getGlobalConfigDir(), 'state');
|
|
275
|
+
}
|
|
276
|
+
|
|
250
277
|
/**
|
|
251
278
|
* Returns the path to the centralized project settings file.
|
|
252
279
|
*/
|
|
@@ -254,9 +281,7 @@ export function getProjectSettingsRegistryPath(): string {
|
|
|
254
281
|
// State lives under pi-research's OWN namespace (~/.pi/research/state), beside
|
|
255
282
|
// config.env and knowledge_db — not in the host pi config dir root (~/.pi/state),
|
|
256
283
|
// which would pollute and risk colliding with the host's own state.
|
|
257
|
-
|
|
258
|
-
?? path.join(getGlobalConfigDir(), 'state');
|
|
259
|
-
return path.join(stateDir, 'project-settings.json');
|
|
284
|
+
return path.join(getStateDir(), 'project-settings.json');
|
|
260
285
|
}
|
|
261
286
|
|
|
262
287
|
/**
|
|
@@ -16,7 +16,6 @@ export interface IStateManager extends IService {
|
|
|
16
16
|
updateHeartbeat(sessionId: string): Promise<void>;
|
|
17
17
|
cleanupStaleSessions(timeoutMs: number): Promise<number>;
|
|
18
18
|
getBrowserServer(): Promise<{ port: number; pid: number; schedulerId?: string; authSecret?: string } | null>;
|
|
19
|
-
setBrowserServer(port: number, pid: number, schedulerId?: string, authSecret?: string): Promise<void>;
|
|
20
19
|
// `expected` makes the clear a compare-and-delete: the entry is only removed when
|
|
21
20
|
// the provided identity fields match, so a stale caller cannot deregister a leader
|
|
22
21
|
// it does not own. Omit only when the caller has just verified ownership under the
|
|
@@ -63,7 +63,11 @@ export type LlmCompleter = (
|
|
|
63
63
|
export async function repairJsonWithLlm<T = any>(
|
|
64
64
|
text: string,
|
|
65
65
|
completer: LlmCompleter,
|
|
66
|
-
|
|
66
|
+
// headers values are string | null to match pi-ai's ProviderHeaders — a null
|
|
67
|
+
// entry suppresses a provider/API default header with that name, and is
|
|
68
|
+
// forwarded to `completer`'s SimpleStreamOptions.headers unchanged, where
|
|
69
|
+
// pi-ai already treats it as documented.
|
|
70
|
+
auth: { apiKey: string; headers?: Record<string, string | null> },
|
|
67
71
|
options: JsonRepairOptions
|
|
68
72
|
): Promise<T | null> {
|
|
69
73
|
const { model, context, schema, serviceName = 'RepairService', signal, maxTokens = 16384, thinkingLevel = 'off' } = options;
|
|
@@ -36,9 +36,15 @@ export interface ModelRegistryLike {
|
|
|
36
36
|
* Structural result of {@link ModelRegistryLike.getApiKeyAndHeaders}. Mirrors the
|
|
37
37
|
* pi-coding-agent `ApiKeyAndHeaders` discriminated union without importing it, so
|
|
38
38
|
* a host skew on the concrete type can't break our build.
|
|
39
|
+
*
|
|
40
|
+
* `headers` values are `string | null` (not just `string`) as of pi 0.84.0's
|
|
41
|
+
* `ProviderHeaders` type — `null` is a legitimate header-DELETION marker (e.g.
|
|
42
|
+
* stripping a placeholder credential before it reaches a gateway), not an
|
|
43
|
+
* absent value. Every consumer of `headers` below must treat `null` as "omit
|
|
44
|
+
* this header," not coerce it to the literal string `"null"`.
|
|
39
45
|
*/
|
|
40
46
|
export type AuthResultLike =
|
|
41
|
-
| { ok: true; apiKey?: string; headers?: Record<string, string> }
|
|
47
|
+
| { ok: true; apiKey?: string; headers?: Record<string, string | null> }
|
|
42
48
|
| { ok: false; error: string };
|
|
43
49
|
|
|
44
50
|
/** Safe `getAll()` — returns [] if the host registry lacks the method. */
|
|
@@ -156,12 +162,16 @@ export async function buildModelRegistry(apiKey?: string, provider?: string): Pr
|
|
|
156
162
|
// with an explicit model (enforced in cli.ts pre-flight), so resolution never
|
|
157
163
|
// falls back to "first available".
|
|
158
164
|
if (apiKey && provider) {
|
|
159
|
-
//
|
|
160
|
-
//
|
|
161
|
-
//
|
|
162
|
-
//
|
|
163
|
-
//
|
|
164
|
-
|
|
165
|
+
// pi 0.84.0 removed `allowNetwork` from setRuntimeApiKey's own options
|
|
166
|
+
// (AuthOperationOptions is now just `{ signal? }`) — but its internal
|
|
167
|
+
// synchronizeCredentialState() now hardcodes `refresh({ allowNetwork:
|
|
168
|
+
// false, ... })` unconditionally on every call, so the "no live catalog
|
|
169
|
+
// connection, no indefinite hang in network-restricted environments"
|
|
170
|
+
// guarantee this call originally needed still holds without asking for
|
|
171
|
+
// it. Confirmed by reading pi-coding-agent's model-runtime.js directly,
|
|
172
|
+
// not just the changelog prose — verify this again on the next pi bump
|
|
173
|
+
// before assuming it's still unconditional.
|
|
174
|
+
await runtime.setRuntimeApiKey(provider, apiKey);
|
|
165
175
|
}
|
|
166
176
|
|
|
167
177
|
// The sync facade all our resolvers consume. getAvailable()/getApiKeyAndHeaders()
|
package/src/core/pi-version.ts
CHANGED
|
@@ -61,8 +61,21 @@ export function compareVersions(a: SemverParts, b: SemverParts): number {
|
|
|
61
61
|
* `ModelRegistry.create()`. `buildModelRegistry` calls `ModelRuntime.create()`
|
|
62
62
|
* unconditionally, and `createAgentSession()` is invoked without the removed
|
|
63
63
|
* `modelRegistry` option, relying on the 0.80.8+ host to build its own.
|
|
64
|
+
*
|
|
65
|
+
* 0.84.0 removed `allowNetwork` from `ModelRuntime.setRuntimeApiKey()`'s own
|
|
66
|
+
* options (`AuthOperationOptions` is now just `{ signal? }`) — but its
|
|
67
|
+
* internal `synchronizeCredentialState()` now hardcodes `refresh({
|
|
68
|
+
* allowNetwork: false, ... })` unconditionally on every call, so
|
|
69
|
+
* `buildModelRegistry`'s explicit-API-key path (model-registry-factory.ts)
|
|
70
|
+
* no longer passes `allowNetwork` at all, RELYING on 0.84.0+'s own
|
|
71
|
+
* unconditional internal guard against a live network catalog connection.
|
|
72
|
+
* On an already-resolved pre-0.84.0 host, `setRuntimeApiKey`'s default
|
|
73
|
+
* (unspecified) network behavior applies instead — reintroducing the
|
|
74
|
+
* indefinite-hang-in-network-restricted-environments bug that explicit
|
|
75
|
+
* option originally existed to prevent. The floor moved to close that gap,
|
|
76
|
+
* not just because the type changed.
|
|
64
77
|
*/
|
|
65
|
-
export const PI_MIN_VERSION: SemverParts = { major: 0, minor:
|
|
78
|
+
export const PI_MIN_VERSION: SemverParts = { major: 0, minor: 84, patch: 0 };
|
|
66
79
|
|
|
67
80
|
/**
|
|
68
81
|
* Newest host line this release was actually exercised against (CI + a real run).
|
|
@@ -70,7 +83,7 @@ export const PI_MIN_VERSION: SemverParts = { major: 0, minor: 80, patch: 8 };
|
|
|
70
83
|
* Compared on MAJOR.MINOR only: a patch bump within a tested line is not a new
|
|
71
84
|
* surface, and warning on it would be noise.
|
|
72
85
|
*/
|
|
73
|
-
export const PI_TESTED_MAX_VERSION: SemverParts = { major: 0, minor:
|
|
86
|
+
export const PI_TESTED_MAX_VERSION: SemverParts = { major: 0, minor: 84, patch: 1 };
|
|
74
87
|
|
|
75
88
|
export type PiCompatibilityLevel = 'ok' | 'unparseable' | 'too-old' | 'untested';
|
|
76
89
|
|
|
@@ -90,6 +90,13 @@ interface ServiceRegistration<T extends IService> {
|
|
|
90
90
|
factory: ServiceFactory<T>;
|
|
91
91
|
instance: T | null;
|
|
92
92
|
initializationPromise: Promise<T> | null;
|
|
93
|
+
// A clear()/replace() disposal currently tearing this service's instance
|
|
94
|
+
// down. get() awaits this before touching registration.instance — without
|
|
95
|
+
// it, a concurrent get() arriving mid-disposal (instance still non-null;
|
|
96
|
+
// dispose() hasn't nulled it yet) would either re-initialize the very
|
|
97
|
+
// instance being disposed (ctx branch) or hand a caller a reference that's
|
|
98
|
+
// about to be (or already was) torn down.
|
|
99
|
+
disposalPromise: Promise<void> | null;
|
|
93
100
|
options: ServiceContainerOptions;
|
|
94
101
|
}
|
|
95
102
|
|
|
@@ -153,6 +160,7 @@ export class ServiceContainer {
|
|
|
153
160
|
factory,
|
|
154
161
|
instance: null,
|
|
155
162
|
initializationPromise: null,
|
|
163
|
+
disposalPromise: null,
|
|
156
164
|
options: mergedOptions,
|
|
157
165
|
});
|
|
158
166
|
|
|
@@ -216,6 +224,16 @@ export class ServiceContainer {
|
|
|
216
224
|
throw new Error(`Service '${name}' is not registered`);
|
|
217
225
|
}
|
|
218
226
|
|
|
227
|
+
// A clear()/replace() disposal in flight must settle before anything below
|
|
228
|
+
// reads registration.instance — otherwise this call could re-initialize
|
|
229
|
+
// (ctx branch) or hand back a reference to the very instance currently
|
|
230
|
+
// being torn down. while(), not if(): a newer disposal can start again
|
|
231
|
+
// while we're awaiting (mirrors the initializationPromise wait pattern
|
|
232
|
+
// used throughout this method).
|
|
233
|
+
while (registration.disposalPromise) {
|
|
234
|
+
await registration.disposalPromise.catch(() => { /* disposal errors are logged by clear()/replace() */ });
|
|
235
|
+
}
|
|
236
|
+
|
|
219
237
|
// Return existing instance if already initialized.
|
|
220
238
|
if (registration.instance) {
|
|
221
239
|
// `_initializeService` publishes `registration.instance` BEFORE awaiting the
|
|
@@ -255,12 +273,30 @@ export class ServiceContainer {
|
|
|
255
273
|
// instance under a running initialize()), and concurrent callers
|
|
256
274
|
// received the instance mid-re-initialization.
|
|
257
275
|
registration.initializationPromise = reinit as Promise<IService>;
|
|
258
|
-
|
|
259
|
-
|
|
276
|
+
// Identity-guarded: only touch shared state if THIS reinit is still the
|
|
277
|
+
// one the registration is currently tracking — a newer reinit/init that
|
|
278
|
+
// superseded it (and already published its own result) must not be
|
|
279
|
+
// clobbered by a stale settlement arriving after it.
|
|
280
|
+
const isStillCurrent = () => registration.initializationPromise === (reinit as Promise<IService>);
|
|
281
|
+
reinit.then(
|
|
282
|
+
() => { if (isStillCurrent()) registration.initializationPromise = null; },
|
|
283
|
+
() => {
|
|
284
|
+
// A re-init failure can leave THIS SAME instance in an inconsistent
|
|
285
|
+
// internal state — e.g. KnowledgeStoreService disposes its old
|
|
286
|
+
// handles before rebuilding on a cwd/mode change, then can still
|
|
287
|
+
// throw mid-rebuild. Unlike a fresh _initializeService() failure
|
|
288
|
+
// (which nulls registration.instance so the next get() rebuilds via
|
|
289
|
+
// the factory), this path used to leave the registry believing the
|
|
290
|
+
// same now-broken object was still a valid instance indefinitely.
|
|
291
|
+
// Mirror _initializeService's own failure handling for consistency.
|
|
292
|
+
if (!isStillCurrent()) return;
|
|
260
293
|
registration.initializationPromise = null;
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
294
|
+
if (registration.instance) {
|
|
295
|
+
registration.instance.lifecycle = ServiceLifecycle.UNINITIALIZED;
|
|
296
|
+
registration.instance = null;
|
|
297
|
+
}
|
|
298
|
+
},
|
|
299
|
+
);
|
|
264
300
|
return reinit;
|
|
265
301
|
}
|
|
266
302
|
return registration.instance as T;
|
|
@@ -344,27 +380,45 @@ export class ServiceContainer {
|
|
|
344
380
|
throw new Error(`Service '${name}' is not registered`);
|
|
345
381
|
}
|
|
346
382
|
|
|
347
|
-
//
|
|
348
|
-
//
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
// concurrent first search.
|
|
352
|
-
if (registration.initializationPromise) {
|
|
353
|
-
await registration.initializationPromise.catch(() => { /* failed init cleaned up after itself */ });
|
|
383
|
+
// Join an already-in-flight clear()/replace() rather than starting a
|
|
384
|
+
// second disposal of the same registration.
|
|
385
|
+
if (registration.disposalPromise) {
|
|
386
|
+
return registration.disposalPromise;
|
|
354
387
|
}
|
|
355
388
|
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
389
|
+
// Published for the whole body (not just the dispose() call below) so a
|
|
390
|
+
// concurrent get() — which now awaits this before touching
|
|
391
|
+
// registration.instance — cannot slip in during the init-settle wait
|
|
392
|
+
// either, not only during the dispose() call itself.
|
|
393
|
+
registration.disposalPromise = (async () => {
|
|
394
|
+
try {
|
|
395
|
+
// Settle an in-flight initialization first. Disposing the early-published
|
|
396
|
+
// instance while its initialize() is still running hands the awaiting get()
|
|
397
|
+
// caller a service that was disposed under it — observed as store-closed
|
|
398
|
+
// errors when /research-config cleared the knowledge store during a
|
|
399
|
+
// concurrent first search.
|
|
400
|
+
if (registration.initializationPromise) {
|
|
401
|
+
await registration.initializationPromise.catch(() => { /* failed init cleaned up after itself */ });
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
if (registration.instance && registration.instance.dispose) {
|
|
405
|
+
await registration.instance.dispose().catch((err: unknown) => {
|
|
406
|
+
logger.warn(`[ServiceContainer] Error disposing service '${name}':`, err);
|
|
407
|
+
});
|
|
408
|
+
}
|
|
361
409
|
|
|
362
|
-
|
|
363
|
-
|
|
410
|
+
registration.instance = null;
|
|
411
|
+
registration.initializationPromise = null;
|
|
364
412
|
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
413
|
+
if (registration.options.enableLogging) {
|
|
414
|
+
logger.debug(`[ServiceContainer] Cleared service '${name}'`);
|
|
415
|
+
}
|
|
416
|
+
} finally {
|
|
417
|
+
registration.disposalPromise = null;
|
|
418
|
+
}
|
|
419
|
+
})();
|
|
420
|
+
|
|
421
|
+
return registration.disposalPromise;
|
|
368
422
|
}
|
|
369
423
|
|
|
370
424
|
/**
|
|
@@ -376,24 +430,38 @@ export class ServiceContainer {
|
|
|
376
430
|
throw new Error(`Service '${name}' is not registered`);
|
|
377
431
|
}
|
|
378
432
|
|
|
379
|
-
//
|
|
380
|
-
if (registration.
|
|
381
|
-
await registration.
|
|
433
|
+
// Join an already-in-flight clear()/replace() — see clear().
|
|
434
|
+
if (registration.disposalPromise) {
|
|
435
|
+
await registration.disposalPromise;
|
|
382
436
|
}
|
|
383
437
|
|
|
384
|
-
//
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
438
|
+
// Published for the whole body — see clear()'s comment for why.
|
|
439
|
+
registration.disposalPromise = (async () => {
|
|
440
|
+
try {
|
|
441
|
+
// Settle an in-flight initialization before swapping — see clear().
|
|
442
|
+
if (registration.initializationPromise) {
|
|
443
|
+
await registration.initializationPromise.catch(() => { /* failed init cleaned up after itself */ });
|
|
444
|
+
}
|
|
390
445
|
|
|
391
|
-
|
|
392
|
-
|
|
446
|
+
// Dispose old instance if present
|
|
447
|
+
if (registration.instance && registration.instance.dispose) {
|
|
448
|
+
await registration.instance.dispose().catch((err: unknown) => {
|
|
449
|
+
logger.warn(`[ServiceContainer] Error disposing old service '${name}':`, err);
|
|
450
|
+
});
|
|
451
|
+
}
|
|
393
452
|
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
453
|
+
registration.instance = newInstance;
|
|
454
|
+
registration.initializationPromise = null;
|
|
455
|
+
|
|
456
|
+
if (registration.options.enableLogging) {
|
|
457
|
+
logger.debug(`[ServiceContainer] Replaced service '${name}'`);
|
|
458
|
+
}
|
|
459
|
+
} finally {
|
|
460
|
+
registration.disposalPromise = null;
|
|
461
|
+
}
|
|
462
|
+
})();
|
|
463
|
+
|
|
464
|
+
return registration.disposalPromise;
|
|
397
465
|
}
|
|
398
466
|
|
|
399
467
|
/**
|
|
@@ -18,6 +18,19 @@ let context: any = null;
|
|
|
18
18
|
let initPromise: Promise<void> | null = null;
|
|
19
19
|
let workerId: string = '';
|
|
20
20
|
|
|
21
|
+
// How many tasks this worker process currently has in flight. poolifier's
|
|
22
|
+
// tasksQueueOptions.concurrency (WORKER_CONCURRENCY, default 2 — see
|
|
23
|
+
// worker-pool-manager.ts) dispatches more than one task at a time to the SAME
|
|
24
|
+
// worker process, and every dispatched task shares this module's browser/
|
|
25
|
+
// context singletons (each gets its own page via createPageSafe(context), but
|
|
26
|
+
// the context/browser themselves are one shared instance). resetBrowser()
|
|
27
|
+
// uses this to avoid tearing that shared instance down out from under a
|
|
28
|
+
// sibling task that is still actively using it — see resetBrowser() below.
|
|
29
|
+
let activeTaskCount = 0;
|
|
30
|
+
// A reset that resetBrowser() deferred because sibling tasks were still
|
|
31
|
+
// active, to be applied once the last one finishes.
|
|
32
|
+
let resetPending = false;
|
|
33
|
+
|
|
21
34
|
/**
|
|
22
35
|
* Set the worker ID for logging purposes
|
|
23
36
|
*/
|
|
@@ -25,6 +38,20 @@ export function setWorkerId(id: string): void {
|
|
|
25
38
|
workerId = id;
|
|
26
39
|
}
|
|
27
40
|
|
|
41
|
+
/** Call when a task begins executing (before it touches the browser/context). */
|
|
42
|
+
export function taskStarted(): void {
|
|
43
|
+
activeTaskCount++;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** Call when a task finishes (success or failure), from a `finally` block. */
|
|
47
|
+
export function taskFinished(): void {
|
|
48
|
+
activeTaskCount = Math.max(0, activeTaskCount - 1);
|
|
49
|
+
if (activeTaskCount === 0 && resetPending) {
|
|
50
|
+
resetPending = false;
|
|
51
|
+
doResetBrowser();
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
28
55
|
/**
|
|
29
56
|
* Check if browser is connected
|
|
30
57
|
*/
|
|
@@ -247,16 +274,38 @@ export function getContext(): any {
|
|
|
247
274
|
return context;
|
|
248
275
|
}
|
|
249
276
|
|
|
250
|
-
|
|
251
|
-
* Reset browser and context (used after crashes)
|
|
252
|
-
*/
|
|
253
|
-
export function resetBrowser(): void {
|
|
277
|
+
function doResetBrowser(): void {
|
|
254
278
|
if (context) Promise.resolve(context.close()).catch((err: Error) => logToDebugFile('DEBUG', `[Worker-${workerId}] Swallowed context close error during reset: ${err.message}`));
|
|
255
279
|
if (browser) Promise.resolve(browser.close()).catch((err: Error) => logToDebugFile('DEBUG', `[Worker-${workerId}] Swallowed browser close error during reset: ${err.message}`));
|
|
256
280
|
context = null;
|
|
257
281
|
browser = null;
|
|
258
282
|
}
|
|
259
283
|
|
|
284
|
+
/**
|
|
285
|
+
* Reset browser and context (used after crashes).
|
|
286
|
+
*
|
|
287
|
+
* A page/session-scoped failure in ONE task (e.g. Playwright/Juggler's
|
|
288
|
+
* generic "Protocol error", thrown for a single crashed tab — not
|
|
289
|
+
* necessarily the whole browser) must not tear down the browser/context a
|
|
290
|
+
* CONCURRENTLY-RUNNING sibling task is still actively using (see
|
|
291
|
+
* activeTaskCount's doc comment): that collaterally fails the sibling's
|
|
292
|
+
* perfectly healthy work with a spurious "browser closed" error, un-retried.
|
|
293
|
+
*
|
|
294
|
+
* If other tasks are still in flight, defer instead of resetting immediately
|
|
295
|
+
* — the deferred reset runs once the last active task finishes (taskFinished
|
|
296
|
+
* above). A GENUINELY dead browser is still caught regardless: initBrowser()'s
|
|
297
|
+
* own isBrowserConnected() check on the very next task re-initializes either
|
|
298
|
+
* way, so deferring costs nothing in that case — it just avoids the
|
|
299
|
+
* collateral damage in the much more common single-crashed-tab case.
|
|
300
|
+
*/
|
|
301
|
+
export function resetBrowser(): void {
|
|
302
|
+
if (activeTaskCount > 1) {
|
|
303
|
+
resetPending = true;
|
|
304
|
+
return;
|
|
305
|
+
}
|
|
306
|
+
doResetBrowser();
|
|
307
|
+
}
|
|
308
|
+
|
|
260
309
|
/**
|
|
261
310
|
* Clean up browser resources with a timeout to prevent hanging
|
|
262
311
|
*/
|
|
@@ -45,7 +45,12 @@ function formatArg(arg) {
|
|
|
45
45
|
return String(arg);
|
|
46
46
|
}
|
|
47
47
|
function redactSecrets(message) {
|
|
48
|
-
let
|
|
48
|
+
let keep = MAX_LOG_MESSAGE_LENGTH;
|
|
49
|
+
if (message.length > keep) {
|
|
50
|
+
const lastKeptUnit = message.charCodeAt(keep - 1);
|
|
51
|
+
if (lastKeptUnit >= 55296 && lastKeptUnit <= 56319) keep -= 1;
|
|
52
|
+
}
|
|
53
|
+
let out = message.length > MAX_LOG_MESSAGE_LENGTH ? `${message.slice(0, keep)}\u2026[truncated ${message.length - keep} chars]` : message;
|
|
49
54
|
out = out.replace(ANSI_PATTERN, "");
|
|
50
55
|
out = out.replace(URL_CREDENTIALS_PATTERN, "$1[REDACTED]@");
|
|
51
56
|
out = out.replace(JWT_PATTERN, "[REDACTED]");
|
|
@@ -1483,6 +1488,7 @@ function isBenignScrapeFailure(error) {
|
|
|
1483
1488
|
const msg = error instanceof Error ? error.message : String(error);
|
|
1484
1489
|
return msg.startsWith("Fetch blocked:") || // bot-protection interstitial (Cloudflare, DDoS-Guard, …)
|
|
1485
1490
|
msg.startsWith("Fetch returned stub:") || // nav-only / near-empty page
|
|
1491
|
+
msg.startsWith("Could not extract content from PDF") || // malformed/encrypted/scanned-only PDF
|
|
1486
1492
|
/^HTTP \d{3}\b/.test(msg) || // remote 4xx/5xx status
|
|
1487
1493
|
isCloudflareBlockError(error) || isTaskTimeoutError(error) || isPoolShutdownError(error);
|
|
1488
1494
|
}
|
|
@@ -2500,9 +2506,21 @@ var browser = null;
|
|
|
2500
2506
|
var context = null;
|
|
2501
2507
|
var initPromise = null;
|
|
2502
2508
|
var workerId3 = "";
|
|
2509
|
+
var activeTaskCount = 0;
|
|
2510
|
+
var resetPending = false;
|
|
2503
2511
|
function setWorkerId3(id) {
|
|
2504
2512
|
workerId3 = id;
|
|
2505
2513
|
}
|
|
2514
|
+
function taskStarted() {
|
|
2515
|
+
activeTaskCount++;
|
|
2516
|
+
}
|
|
2517
|
+
function taskFinished() {
|
|
2518
|
+
activeTaskCount = Math.max(0, activeTaskCount - 1);
|
|
2519
|
+
if (activeTaskCount === 0 && resetPending) {
|
|
2520
|
+
resetPending = false;
|
|
2521
|
+
doResetBrowser();
|
|
2522
|
+
}
|
|
2523
|
+
}
|
|
2506
2524
|
var isBrowserConnected = () => {
|
|
2507
2525
|
try {
|
|
2508
2526
|
return browser && typeof browser.isConnected === "function" && browser.isConnected();
|
|
@@ -2628,12 +2646,19 @@ async function initBrowser() {
|
|
|
2628
2646
|
function getContext() {
|
|
2629
2647
|
return context;
|
|
2630
2648
|
}
|
|
2631
|
-
function
|
|
2649
|
+
function doResetBrowser() {
|
|
2632
2650
|
if (context) Promise.resolve(context.close()).catch((err) => logToDebugFile3("DEBUG", `[Worker-${workerId3}] Swallowed context close error during reset: ${err.message}`));
|
|
2633
2651
|
if (browser) Promise.resolve(browser.close()).catch((err) => logToDebugFile3("DEBUG", `[Worker-${workerId3}] Swallowed browser close error during reset: ${err.message}`));
|
|
2634
2652
|
context = null;
|
|
2635
2653
|
browser = null;
|
|
2636
2654
|
}
|
|
2655
|
+
function resetBrowser() {
|
|
2656
|
+
if (activeTaskCount > 1) {
|
|
2657
|
+
resetPending = true;
|
|
2658
|
+
return;
|
|
2659
|
+
}
|
|
2660
|
+
doResetBrowser();
|
|
2661
|
+
}
|
|
2637
2662
|
async function cleanupBrowser2() {
|
|
2638
2663
|
const timeoutMs = 2e3;
|
|
2639
2664
|
const cleanup = async () => {
|
|
@@ -2695,6 +2720,7 @@ async function runTask(data) {
|
|
|
2695
2720
|
if (taskTimeoutMs && taskTimeoutMs > 0) {
|
|
2696
2721
|
taskTimer = setTimeout(() => abortController.abort(new Error(`Task timed out after ${taskTimeoutMs}ms`)), taskTimeoutMs);
|
|
2697
2722
|
}
|
|
2723
|
+
taskStarted();
|
|
2698
2724
|
try {
|
|
2699
2725
|
await initBrowser();
|
|
2700
2726
|
const initMs = Date.now() - startTime;
|
|
@@ -2731,6 +2757,7 @@ async function runTask(data) {
|
|
|
2731
2757
|
};
|
|
2732
2758
|
} finally {
|
|
2733
2759
|
if (taskTimer !== void 0) clearTimeout(taskTimer);
|
|
2760
|
+
taskFinished();
|
|
2734
2761
|
}
|
|
2735
2762
|
}
|
|
2736
2763
|
var worker = new ClusterWorker(runTask, {
|
|
@@ -26,6 +26,8 @@ import {
|
|
|
26
26
|
getContext,
|
|
27
27
|
resetBrowser,
|
|
28
28
|
cleanupBrowser,
|
|
29
|
+
taskStarted,
|
|
30
|
+
taskFinished,
|
|
29
31
|
} from './thread-worker-browser.ts';
|
|
30
32
|
import {
|
|
31
33
|
setWorkerId as setMessagingWorkerId,
|
|
@@ -102,6 +104,11 @@ async function runTask(data: TaskData | undefined): Promise<TaskResult> {
|
|
|
102
104
|
taskTimer = setTimeout(() => abortController.abort(new Error(`Task timed out after ${taskTimeoutMs}ms`)), taskTimeoutMs);
|
|
103
105
|
}
|
|
104
106
|
|
|
107
|
+
// Registered around the whole browser-touching lifetime of this task so
|
|
108
|
+
// resetBrowser() (below) can tell whether a sibling task dispatched to this
|
|
109
|
+
// same worker process (WORKER_CONCURRENCY > 1) is still using the shared
|
|
110
|
+
// browser/context before tearing it down.
|
|
111
|
+
taskStarted();
|
|
105
112
|
try {
|
|
106
113
|
await initBrowser();
|
|
107
114
|
const initMs = Date.now() - startTime;
|
|
@@ -149,6 +156,7 @@ async function runTask(data: TaskData | undefined): Promise<TaskResult> {
|
|
|
149
156
|
};
|
|
150
157
|
} finally {
|
|
151
158
|
if (taskTimer !== undefined) clearTimeout(taskTimer);
|
|
159
|
+
taskFinished();
|
|
152
160
|
}
|
|
153
161
|
}
|
|
154
162
|
|