@loopover/miner 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DEPLOYMENT.md +156 -0
- package/Dockerfile +38 -0
- package/README.md +269 -0
- package/bin/gittensory-miner-mcp.d.ts +64 -0
- package/bin/gittensory-miner-mcp.js +279 -0
- package/bin/gittensory-miner.js +196 -0
- package/docs/coding-agent-driver.md +140 -0
- package/docs/config-precedence.md +96 -0
- package/docs/cross-repo-discovery-phase1.md +55 -0
- package/docs/discovery-index-contract.md +60 -0
- package/docs/discovery-plane-operator-guide.md +101 -0
- package/docs/env-reference.md +30 -0
- package/docs/fleet-run-manifest.md +50 -0
- package/docs/miner-goal-spec.md +76 -0
- package/docs/miner-selfimprove-calibration.md +105 -0
- package/docs/observability.md +60 -0
- package/docs/operations-runbook.md +209 -0
- package/docs/repo-agnostic-capability-audit.md +129 -0
- package/docs/unattended-scheduling.md +107 -0
- package/expected-engine.version +1 -0
- package/lib/ams-policy.d.ts +20 -0
- package/lib/ams-policy.js +68 -0
- package/lib/attempt-cli.d.ts +99 -0
- package/lib/attempt-cli.js +523 -0
- package/lib/attempt-input-builder.d.ts +24 -0
- package/lib/attempt-input-builder.js +91 -0
- package/lib/attempt-log.d.ts +37 -0
- package/lib/attempt-log.js +177 -0
- package/lib/attempt-runner.d.ts +59 -0
- package/lib/attempt-runner.js +164 -0
- package/lib/attempt-worktree.d.ts +31 -0
- package/lib/attempt-worktree.js +94 -0
- package/lib/calibration-cli.d.ts +1 -0
- package/lib/calibration-cli.js +92 -0
- package/lib/calibration-run.d.ts +161 -0
- package/lib/calibration-run.js +232 -0
- package/lib/calibration-types.d.ts +46 -0
- package/lib/calibration-types.js +74 -0
- package/lib/calibration.d.ts +24 -0
- package/lib/calibration.js +99 -0
- package/lib/ci-poller.d.ts +33 -0
- package/lib/ci-poller.js +234 -0
- package/lib/claim-adjudication.d.ts +21 -0
- package/lib/claim-adjudication.js +36 -0
- package/lib/claim-conflict-resolver.d.ts +29 -0
- package/lib/claim-conflict-resolver.js +101 -0
- package/lib/claim-ledger-cli.d.ts +59 -0
- package/lib/claim-ledger-cli.js +320 -0
- package/lib/claim-ledger-expiry.d.ts +20 -0
- package/lib/claim-ledger-expiry.js +41 -0
- package/lib/claim-ledger.d.ts +63 -0
- package/lib/claim-ledger.js +303 -0
- package/lib/cli-error.d.ts +3 -0
- package/lib/cli-error.js +27 -0
- package/lib/cli.d.ts +3 -0
- package/lib/cli.js +72 -0
- package/lib/coding-agent-construction.d.ts +17 -0
- package/lib/coding-agent-construction.js +103 -0
- package/lib/coding-agent-house-rules.d.ts +24 -0
- package/lib/coding-agent-house-rules.js +64 -0
- package/lib/coding-task-spec.d.ts +44 -0
- package/lib/coding-task-spec.js +183 -0
- package/lib/config-precedence.d.ts +25 -0
- package/lib/config-precedence.js +38 -0
- package/lib/deny-check.d.ts +11 -0
- package/lib/deny-check.js +76 -0
- package/lib/deny-hook-synthesis.d.ts +93 -0
- package/lib/deny-hook-synthesis.js +399 -0
- package/lib/deny-hooks.d.ts +28 -0
- package/lib/deny-hooks.js +155 -0
- package/lib/deployment-docs-audit.d.ts +41 -0
- package/lib/deployment-docs-audit.js +115 -0
- package/lib/discover-cli.d.ts +94 -0
- package/lib/discover-cli.js +284 -0
- package/lib/discovery-throttle.d.ts +8 -0
- package/lib/discovery-throttle.js +33 -0
- package/lib/env-file-indirection.d.ts +4 -0
- package/lib/env-file-indirection.js +45 -0
- package/lib/event-ledger-cli.d.ts +82 -0
- package/lib/event-ledger-cli.js +266 -0
- package/lib/event-ledger.d.ts +37 -0
- package/lib/event-ledger.js +210 -0
- package/lib/execute-local-write.d.ts +14 -0
- package/lib/execute-local-write.js +50 -0
- package/lib/feasibility-cli.d.ts +25 -0
- package/lib/feasibility-cli.js +80 -0
- package/lib/forge-config.d.ts +17 -0
- package/lib/forge-config.js +37 -0
- package/lib/governor-action-mode.d.ts +26 -0
- package/lib/governor-action-mode.js +47 -0
- package/lib/governor-chokepoint-persisted.d.ts +18 -0
- package/lib/governor-chokepoint-persisted.js +46 -0
- package/lib/governor-chokepoint.d.ts +14 -0
- package/lib/governor-chokepoint.js +52 -0
- package/lib/governor-kill-switch.d.ts +26 -0
- package/lib/governor-kill-switch.js +47 -0
- package/lib/governor-ledger-cli.d.ts +32 -0
- package/lib/governor-ledger-cli.js +155 -0
- package/lib/governor-ledger.d.ts +46 -0
- package/lib/governor-ledger.js +203 -0
- package/lib/governor-open-pr.d.ts +21 -0
- package/lib/governor-open-pr.js +27 -0
- package/lib/governor-pause-cli.d.ts +23 -0
- package/lib/governor-pause-cli.js +171 -0
- package/lib/governor-run-halt.d.ts +37 -0
- package/lib/governor-run-halt.js +58 -0
- package/lib/governor-state.d.ts +63 -0
- package/lib/governor-state.js +363 -0
- package/lib/governor-write-rate-limit.d.ts +30 -0
- package/lib/governor-write-rate-limit.js +64 -0
- package/lib/harness-submission-trigger.d.ts +69 -0
- package/lib/harness-submission-trigger.js +138 -0
- package/lib/http-retry.d.ts +12 -0
- package/lib/http-retry.js +52 -0
- package/lib/laptop-init.d.ts +53 -0
- package/lib/laptop-init.js +332 -0
- package/lib/live-issue-snapshot.d.ts +16 -0
- package/lib/live-issue-snapshot.js +117 -0
- package/lib/local-store.d.ts +18 -0
- package/lib/local-store.js +62 -0
- package/lib/logger.d.ts +58 -0
- package/lib/logger.js +167 -0
- package/lib/loop-cli.d.ts +66 -0
- package/lib/loop-cli.js +540 -0
- package/lib/loop-closure.d.ts +35 -0
- package/lib/loop-closure.js +66 -0
- package/lib/loop-reentry.d.ts +51 -0
- package/lib/loop-reentry.js +124 -0
- package/lib/manage-poll.d.ts +79 -0
- package/lib/manage-poll.js +254 -0
- package/lib/manage-status.d.ts +70 -0
- package/lib/manage-status.js +241 -0
- package/lib/metrics-cli.d.ts +9 -0
- package/lib/metrics-cli.js +50 -0
- package/lib/migrate-cli.d.ts +24 -0
- package/lib/migrate-cli.js +111 -0
- package/lib/miner-goal-spec.d.ts +12 -0
- package/lib/miner-goal-spec.js +67 -0
- package/lib/opportunity-fanout.d.ts +86 -0
- package/lib/opportunity-fanout.js +597 -0
- package/lib/opportunity-ranker.d.ts +36 -0
- package/lib/opportunity-ranker.js +118 -0
- package/lib/orb-export.d.ts +54 -0
- package/lib/orb-export.js +206 -0
- package/lib/plan-store-cli.d.ts +37 -0
- package/lib/plan-store-cli.js +151 -0
- package/lib/plan-store.d.ts +51 -0
- package/lib/plan-store.js +231 -0
- package/lib/policy-doc-cache.d.ts +25 -0
- package/lib/policy-doc-cache.js +79 -0
- package/lib/policy-verdict-cache.d.ts +35 -0
- package/lib/policy-verdict-cache.js +106 -0
- package/lib/portfolio-dashboard.d.ts +30 -0
- package/lib/portfolio-dashboard.js +105 -0
- package/lib/portfolio-discovery.d.ts +29 -0
- package/lib/portfolio-discovery.js +100 -0
- package/lib/portfolio-queue-cli.d.ts +87 -0
- package/lib/portfolio-queue-cli.js +509 -0
- package/lib/portfolio-queue-expiry.d.ts +20 -0
- package/lib/portfolio-queue-expiry.js +51 -0
- package/lib/portfolio-queue-manager.d.ts +50 -0
- package/lib/portfolio-queue-manager.js +134 -0
- package/lib/portfolio-queue.d.ts +62 -0
- package/lib/portfolio-queue.js +351 -0
- package/lib/pr-disposition-poller.d.ts +26 -0
- package/lib/pr-disposition-poller.js +163 -0
- package/lib/pr-number-parse.d.ts +4 -0
- package/lib/pr-number-parse.js +23 -0
- package/lib/pr-outcome.d.ts +41 -0
- package/lib/pr-outcome.js +90 -0
- package/lib/prediction-ledger.d.ts +47 -0
- package/lib/prediction-ledger.js +222 -0
- package/lib/pretooluse-hook.d.ts +31 -0
- package/lib/pretooluse-hook.js +93 -0
- package/lib/process-lifecycle.d.ts +31 -0
- package/lib/process-lifecycle.js +106 -0
- package/lib/purge-cli.d.ts +44 -0
- package/lib/purge-cli.js +198 -0
- package/lib/rejection-signal.d.ts +6 -0
- package/lib/rejection-signal.js +101 -0
- package/lib/rejection-state-machine.d.ts +37 -0
- package/lib/rejection-state-machine.js +81 -0
- package/lib/rejection-templates.d.ts +12 -0
- package/lib/rejection-templates.js +71 -0
- package/lib/replay-objective-anchor.d.ts +79 -0
- package/lib/replay-objective-anchor.js +179 -0
- package/lib/replay-snapshot.d.ts +41 -0
- package/lib/replay-snapshot.js +288 -0
- package/lib/replay-task-generation.d.ts +126 -0
- package/lib/replay-task-generation.js +223 -0
- package/lib/repo-clone.d.ts +19 -0
- package/lib/repo-clone.js +102 -0
- package/lib/run-state-cli.d.ts +27 -0
- package/lib/run-state-cli.js +154 -0
- package/lib/run-state.d.ts +37 -0
- package/lib/run-state.js +150 -0
- package/lib/schema-version.d.ts +17 -0
- package/lib/schema-version.js +71 -0
- package/lib/self-review-context.d.ts +28 -0
- package/lib/self-review-context.js +339 -0
- package/lib/slop-assessment.d.ts +3 -0
- package/lib/slop-assessment.js +20 -0
- package/lib/stack-detection.d.ts +41 -0
- package/lib/stack-detection.js +248 -0
- package/lib/status.d.ts +64 -0
- package/lib/status.js +441 -0
- package/lib/store-maintenance.d.ts +31 -0
- package/lib/store-maintenance.js +183 -0
- package/lib/submission-freshness-check.d.ts +32 -0
- package/lib/submission-freshness-check.js +93 -0
- package/lib/update-check.d.ts +36 -0
- package/lib/update-check.js +161 -0
- package/lib/version.d.ts +3 -0
- package/lib/version.js +10 -0
- package/lib/worktree-allocator.d.ts +39 -0
- package/lib/worktree-allocator.js +262 -0
- package/package.json +50 -0
- package/schema/miner-goal-spec.schema.json +111 -0
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { QueueEntry, QueueLeaseEntry } from "./portfolio-queue.js";
|
|
2
|
+
|
|
3
|
+
export declare const DEFAULT_MAX_LEASE_MS: number;
|
|
4
|
+
|
|
5
|
+
export type PortfolioQueueExpiryStore = {
|
|
6
|
+
listInProgress(): QueueLeaseEntry[];
|
|
7
|
+
reclaimStuckItem(repoFullName: string, identifier: string): QueueEntry | null;
|
|
8
|
+
};
|
|
9
|
+
|
|
10
|
+
export function findStuckItems(
|
|
11
|
+
items: QueueLeaseEntry[],
|
|
12
|
+
nowMs: number,
|
|
13
|
+
maxLeaseMs: number,
|
|
14
|
+
): QueueLeaseEntry[];
|
|
15
|
+
|
|
16
|
+
export function sweepStuckItems(
|
|
17
|
+
store: PortfolioQueueExpiryStore,
|
|
18
|
+
nowMs: number,
|
|
19
|
+
maxLeaseMs?: number,
|
|
20
|
+
): QueueEntry[];
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/** PURE — no IO, no Date, no random (#4827). Mirror of claim-ledger-expiry.js for the portfolio-queue store: a
|
|
2
|
+
* crashed/killed process leaves its item stuck 'in_progress' forever, so sweep leases older than a bound back to
|
|
3
|
+
* 'queued'. */
|
|
4
|
+
|
|
5
|
+
// A generous default: a real attempt rarely holds a single portfolio item for long, so 30 minutes without the row
|
|
6
|
+
// leaving 'in_progress' strongly implies the owning process died rather than that it is still working.
|
|
7
|
+
export const DEFAULT_MAX_LEASE_MS = 30 * 60 * 1000;
|
|
8
|
+
|
|
9
|
+
function leaseAgeMs(item, nowMs) {
|
|
10
|
+
const leasedAtMs = Date.parse(item.leasedAt);
|
|
11
|
+
if (!Number.isFinite(leasedAtMs)) return null;
|
|
12
|
+
return nowMs - leasedAtMs;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Return in-flight items whose lease age is strictly greater than `maxLeaseMs`. An item whose age equals
|
|
17
|
+
* `maxLeaseMs` exactly is still within the window (not stuck). Items that are not 'in_progress', or whose
|
|
18
|
+
* `leasedAt` is missing/unparseable, are never returned.
|
|
19
|
+
*/
|
|
20
|
+
export function findStuckItems(items, nowMs, maxLeaseMs) {
|
|
21
|
+
if (!Number.isFinite(nowMs) || nowMs < 0) throw new Error("invalid_now_ms");
|
|
22
|
+
if (!Number.isFinite(maxLeaseMs) || maxLeaseMs < 0) throw new Error("invalid_max_lease_ms");
|
|
23
|
+
if (!Array.isArray(items)) throw new Error("invalid_items");
|
|
24
|
+
|
|
25
|
+
const stuck = [];
|
|
26
|
+
for (const item of items) {
|
|
27
|
+
if (item?.status !== "in_progress") continue;
|
|
28
|
+
const ageMs = leaseAgeMs(item, nowMs);
|
|
29
|
+
if (ageMs === null) continue;
|
|
30
|
+
if (ageMs > maxLeaseMs) stuck.push(item);
|
|
31
|
+
}
|
|
32
|
+
return stuck;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Reclaim every stuck in-flight item back to 'queued', returning the reclaimed entries. `store.listInProgress()`
|
|
37
|
+
* supplies the lease-annotated rows and `store.reclaimStuckItem()` performs the atomic per-item flip — the same
|
|
38
|
+
* store/sweep split sweepExpiredClaims uses.
|
|
39
|
+
*/
|
|
40
|
+
export function sweepStuckItems(store, nowMs, maxLeaseMs = DEFAULT_MAX_LEASE_MS) {
|
|
41
|
+
const inProgress = store.listInProgress();
|
|
42
|
+
const stuck = findStuckItems(inProgress, nowMs, maxLeaseMs);
|
|
43
|
+
const reclaimed = [];
|
|
44
|
+
for (const item of stuck) {
|
|
45
|
+
// Echo the item's OWN apiBaseUrl back (#5563) rather than defaulting: two forge hosts can each have an
|
|
46
|
+
// in-flight item with the same owner/repo+identifier, and defaulting here would reclaim the wrong host's row.
|
|
47
|
+
const updated = store.reclaimStuckItem(item.repoFullName, item.identifier, item.apiBaseUrl);
|
|
48
|
+
if (updated) reclaimed.push(updated);
|
|
49
|
+
}
|
|
50
|
+
return reclaimed;
|
|
51
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import type { PortfolioCaps } from "@loopover/engine";
|
|
2
|
+
import type { EnqueueItem, PortfolioQueueStore, QueueEntry } from "./portfolio-queue.js";
|
|
3
|
+
|
|
4
|
+
export type PortfolioQueueClaimTarget = {
|
|
5
|
+
apiBaseUrl: string;
|
|
6
|
+
repoFullName: string;
|
|
7
|
+
identifier: string;
|
|
8
|
+
};
|
|
9
|
+
|
|
10
|
+
export function queueItemId(apiBaseUrl: string, repoFullName: string, identifier: string): string;
|
|
11
|
+
|
|
12
|
+
export function parseQueueItemId(id: string): PortfolioQueueClaimTarget;
|
|
13
|
+
|
|
14
|
+
export function normalizePortfolioCaps(caps?: Partial<PortfolioCaps>): PortfolioCaps;
|
|
15
|
+
|
|
16
|
+
export function entriesToPortfolioQueue(entries: QueueEntry[]): {
|
|
17
|
+
buckets: Array<{
|
|
18
|
+
repoFullName: string;
|
|
19
|
+
items: Array<{ id: string; repoFullName: string; state: "queued" | "in_progress" }>;
|
|
20
|
+
}>;
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
export function selectEligibleBatch(
|
|
24
|
+
entries: QueueEntry[],
|
|
25
|
+
caps: PortfolioCaps,
|
|
26
|
+
): PortfolioQueueClaimTarget[];
|
|
27
|
+
|
|
28
|
+
export type PortfolioQueueManager = {
|
|
29
|
+
caps: PortfolioCaps;
|
|
30
|
+
store: PortfolioQueueStore;
|
|
31
|
+
dbPath: string;
|
|
32
|
+
enqueue(item: EnqueueItem): QueueEntry;
|
|
33
|
+
listQueue(repoFullName?: string | null): QueueEntry[];
|
|
34
|
+
markDone(repoFullName: string, identifier: string, apiBaseUrl?: string): QueueEntry | null;
|
|
35
|
+
markFailed(repoFullName: string, identifier: string, apiBaseUrl?: string): QueueEntry | null;
|
|
36
|
+
reclaimStuckItems(maxLeaseMs?: number): QueueEntry[];
|
|
37
|
+
claimNextBatch(): QueueEntry[];
|
|
38
|
+
close(): void;
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
export type InitPortfolioQueueManagerOptions = {
|
|
42
|
+
caps?: Partial<PortfolioCaps>;
|
|
43
|
+
store?: PortfolioQueueStore;
|
|
44
|
+
dbPath?: string;
|
|
45
|
+
staleLeaseMs?: number;
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
export function initPortfolioQueueManager(options?: InitPortfolioQueueManagerOptions): PortfolioQueueManager;
|
|
49
|
+
|
|
50
|
+
export function closeDefaultPortfolioQueueManager(): void;
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
// Stateful PortfolioQueueManager (#4285): compose the persisted SQLite portfolio/queue store
|
|
2
|
+
// (portfolio-queue.js, #2292) with the pure engine selector (nextEligibleItems, queue.ts, #2326) so batch
|
|
3
|
+
// claiming respects global/per-repo WIP caps and cross-repo diversification instead of a naive priority-only
|
|
4
|
+
// single-row dequeue. Caps are plain constructor arguments — not wired to .gittensory-miner.yml here.
|
|
5
|
+
import { nextEligibleItems } from "@loopover/engine";
|
|
6
|
+
import { DEFAULT_FORGE_CONFIG } from "./forge-config.js";
|
|
7
|
+
import { initPortfolioQueueStore } from "./portfolio-queue.js";
|
|
8
|
+
import { DEFAULT_MAX_LEASE_MS, sweepStuckItems } from "./portfolio-queue-expiry.js";
|
|
9
|
+
|
|
10
|
+
const ITEM_ID_SEPARATOR = "::";
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Stable composite id for projecting SQLite rows into the engine's PortfolioQueueItem shape. Encodes apiBaseUrl
|
|
14
|
+
* too (#5563) — the engine's own selection logic has no forge dimension, but two hosts can now enqueue an item
|
|
15
|
+
* under the same repoFullName+identifier (post-#5563 scoping), and the id is the ONLY thing selectEligibleBatch's
|
|
16
|
+
* output threads back to batchClaim; without the host baked in here, a selected item's host would be lost and
|
|
17
|
+
* batchClaim would default to github.com, potentially claiming a DIFFERENT row than the one the engine selected.
|
|
18
|
+
*/
|
|
19
|
+
export function queueItemId(apiBaseUrl, repoFullName, identifier) {
|
|
20
|
+
return `${apiBaseUrl}${ITEM_ID_SEPARATOR}${repoFullName}${ITEM_ID_SEPARATOR}${identifier}`;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/** Reverse {@link queueItemId} after engine selection so claims can target SQLite primary keys. */
|
|
24
|
+
export function parseQueueItemId(id) {
|
|
25
|
+
if (typeof id !== "string") throw new Error("invalid_queue_item_id");
|
|
26
|
+
const firstSeparatorIndex = id.indexOf(ITEM_ID_SEPARATOR);
|
|
27
|
+
if (firstSeparatorIndex <= 0) throw new Error("invalid_queue_item_id");
|
|
28
|
+
const rest = id.slice(firstSeparatorIndex + ITEM_ID_SEPARATOR.length);
|
|
29
|
+
const secondSeparatorIndex = rest.indexOf(ITEM_ID_SEPARATOR);
|
|
30
|
+
if (secondSeparatorIndex <= 0 || secondSeparatorIndex === rest.length - ITEM_ID_SEPARATOR.length) {
|
|
31
|
+
throw new Error("invalid_queue_item_id");
|
|
32
|
+
}
|
|
33
|
+
return {
|
|
34
|
+
apiBaseUrl: id.slice(0, firstSeparatorIndex),
|
|
35
|
+
repoFullName: rest.slice(0, secondSeparatorIndex),
|
|
36
|
+
identifier: rest.slice(secondSeparatorIndex + ITEM_ID_SEPARATOR.length),
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** Coerce caps to finite non-negative integers (mirrors the engine's normalizeCaps posture). */
|
|
41
|
+
export function normalizePortfolioCaps(caps = {}) {
|
|
42
|
+
const globalWipCap = Number.isFinite(caps.globalWipCap) ? Math.max(0, Math.trunc(caps.globalWipCap)) : 0;
|
|
43
|
+
const perRepoWipCap = Number.isFinite(caps.perRepoWipCap) ? Math.max(0, Math.trunc(caps.perRepoWipCap)) : 0;
|
|
44
|
+
return { globalWipCap, perRepoWipCap };
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** Project persisted queue rows into the engine's in-memory PortfolioQueue (done rows omitted). Pure. */
|
|
48
|
+
export function entriesToPortfolioQueue(entries) {
|
|
49
|
+
const activeEntries = Array.isArray(entries) ? entries.filter((entry) => entry?.status !== "done") : [];
|
|
50
|
+
const bucketsByRepo = new Map();
|
|
51
|
+
const bucketOrder = [];
|
|
52
|
+
for (const entry of activeEntries) {
|
|
53
|
+
const repoFullName = typeof entry.repoFullName === "string" ? entry.repoFullName.trim() : "";
|
|
54
|
+
const identifier = typeof entry.identifier === "string" ? entry.identifier.trim() : "";
|
|
55
|
+
if (!repoFullName || !identifier) continue;
|
|
56
|
+
// Falls back to the github.com default (matching every store's own normalizeApiBaseUrl) so a row from
|
|
57
|
+
// before #5563 threaded apiBaseUrl through this fold still gets a valid, host-scoped id.
|
|
58
|
+
const apiBaseUrl = typeof entry.apiBaseUrl === "string" && entry.apiBaseUrl.trim() ? entry.apiBaseUrl.trim() : DEFAULT_FORGE_CONFIG.apiBaseUrl;
|
|
59
|
+
const repoKey = repoFullName.toLowerCase();
|
|
60
|
+
if (!bucketsByRepo.has(repoKey)) {
|
|
61
|
+
bucketsByRepo.set(repoKey, []);
|
|
62
|
+
bucketOrder.push(repoKey);
|
|
63
|
+
}
|
|
64
|
+
bucketsByRepo.get(repoKey).push({
|
|
65
|
+
id: queueItemId(apiBaseUrl, repoFullName, identifier),
|
|
66
|
+
repoFullName,
|
|
67
|
+
state: entry.status === "in_progress" ? "in_progress" : "queued",
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
return {
|
|
71
|
+
buckets: bucketOrder.map((repoFullName) => ({
|
|
72
|
+
repoFullName,
|
|
73
|
+
items: bucketsByRepo.get(repoFullName),
|
|
74
|
+
})),
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** Select the next eligible batch from active rows using the engine primitive. Pure. */
|
|
79
|
+
export function selectEligibleBatch(entries, caps) {
|
|
80
|
+
const normalizedCaps = normalizePortfolioCaps(caps);
|
|
81
|
+
const queue = entriesToPortfolioQueue(entries);
|
|
82
|
+
return nextEligibleItems(queue, normalizedCaps).map((item) => parseQueueItemId(item.id));
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Open a caps-aware portfolio queue manager backed by the local SQLite store. The existing single-row
|
|
87
|
+
* `dequeueNext()` CLI surface is untouched — this adds `claimNextBatch()` for fleet-style batch claiming.
|
|
88
|
+
*/
|
|
89
|
+
export function initPortfolioQueueManager(options = {}) {
|
|
90
|
+
const caps = normalizePortfolioCaps(options.caps ?? { globalWipCap: 1, perRepoWipCap: 1 });
|
|
91
|
+
const store = options.store ?? initPortfolioQueueStore(options.dbPath);
|
|
92
|
+
// A lease older than this means the process that claimed the item almost certainly died; the item is swept back
|
|
93
|
+
// to 'queued' so it no longer occupies WIP capacity forever (#4827).
|
|
94
|
+
const staleLeaseMs = Number.isFinite(options.staleLeaseMs) ? options.staleLeaseMs : DEFAULT_MAX_LEASE_MS;
|
|
95
|
+
|
|
96
|
+
return {
|
|
97
|
+
caps,
|
|
98
|
+
store,
|
|
99
|
+
dbPath: store.dbPath,
|
|
100
|
+
enqueue(item) {
|
|
101
|
+
return store.enqueue(item);
|
|
102
|
+
},
|
|
103
|
+
listQueue(repoFullName) {
|
|
104
|
+
return store.listQueue(repoFullName);
|
|
105
|
+
},
|
|
106
|
+
markDone(repoFullName, identifier, apiBaseUrl) {
|
|
107
|
+
return store.markDone(repoFullName, identifier, apiBaseUrl);
|
|
108
|
+
},
|
|
109
|
+
markFailed(repoFullName, identifier, apiBaseUrl) {
|
|
110
|
+
return store.markFailed(repoFullName, identifier, apiBaseUrl);
|
|
111
|
+
},
|
|
112
|
+
/** Sweep leases orphaned by a crashed/killed process back to 'queued', returning the reclaimed items (#4827). */
|
|
113
|
+
reclaimStuckItems(maxLeaseMs = staleLeaseMs) {
|
|
114
|
+
return sweepStuckItems(store, Date.now(), maxLeaseMs);
|
|
115
|
+
},
|
|
116
|
+
// The engine primitive itself (@loopover/engine's nextEligibleItems) has no apiBaseUrl concept --
|
|
117
|
+
// it only ever sees the opaque `id` string. queueItemId/parseQueueItemId (#5563) smuggle the host through
|
|
118
|
+
// that id round-trip, so selectFn's output below correctly carries each selected item's OWN apiBaseUrl into
|
|
119
|
+
// batchClaim, instead of every claim defaulting to github.com regardless of which host's row was selected.
|
|
120
|
+
claimNextBatch() {
|
|
121
|
+
// Reclaim orphaned leases first, so an item stranded 'in_progress' by a dead process becomes eligible again
|
|
122
|
+
// instead of permanently consuming a WIP slot and starving the queue.
|
|
123
|
+
sweepStuckItems(store, Date.now(), staleLeaseMs);
|
|
124
|
+
return store.batchClaim((entries) => selectEligibleBatch(entries, caps));
|
|
125
|
+
},
|
|
126
|
+
close() {
|
|
127
|
+
store.close();
|
|
128
|
+
},
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
export function closeDefaultPortfolioQueueManager() {
|
|
133
|
+
// Reserved for symmetry with other miner stores; managers are opened explicitly today.
|
|
134
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
export type QueueStatus = "queued" | "in_progress" | "done";
|
|
2
|
+
|
|
3
|
+
export type QueueEntry = {
|
|
4
|
+
apiBaseUrl: string;
|
|
5
|
+
repoFullName: string;
|
|
6
|
+
identifier: string;
|
|
7
|
+
priority: number;
|
|
8
|
+
status: QueueStatus;
|
|
9
|
+
enqueuedAt: string;
|
|
10
|
+
};
|
|
11
|
+
|
|
12
|
+
export type EnqueueItem = {
|
|
13
|
+
repoFullName: string;
|
|
14
|
+
identifier: string;
|
|
15
|
+
priority?: number | null;
|
|
16
|
+
apiBaseUrl?: string;
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
/** Lease-annotated view of an in-flight row: when it was claimed, for the expiry sweep (#4827). */
|
|
20
|
+
export type QueueLeaseEntry = {
|
|
21
|
+
apiBaseUrl: string;
|
|
22
|
+
repoFullName: string;
|
|
23
|
+
identifier: string;
|
|
24
|
+
status: QueueStatus;
|
|
25
|
+
leasedAt: string | null;
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
export type PortfolioQueueStore = {
|
|
29
|
+
dbPath: string;
|
|
30
|
+
enqueue(item: EnqueueItem): QueueEntry;
|
|
31
|
+
dequeueNext(): QueueEntry | null;
|
|
32
|
+
listQueue(repoFullName?: string | null): QueueEntry[];
|
|
33
|
+
listInProgress(): QueueLeaseEntry[];
|
|
34
|
+
markDone(repoFullName: string, identifier: string, apiBaseUrl?: string): QueueEntry | null;
|
|
35
|
+
markFailed(repoFullName: string, identifier: string, apiBaseUrl?: string): QueueEntry | null;
|
|
36
|
+
reclaimStuckItem(repoFullName: string, identifier: string, apiBaseUrl?: string): QueueEntry | null;
|
|
37
|
+
requeueItem(repoFullName: string, identifier: string, apiBaseUrl?: string): QueueEntry | null;
|
|
38
|
+
batchClaim(
|
|
39
|
+
selectFn: (
|
|
40
|
+
entries: QueueEntry[],
|
|
41
|
+
) => Array<{ repoFullName: string; identifier: string; apiBaseUrl?: string }>,
|
|
42
|
+
): QueueEntry[];
|
|
43
|
+
close(): void;
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
export const QUEUE_STATUSES: readonly QueueStatus[];
|
|
47
|
+
|
|
48
|
+
export function resolvePortfolioQueueDbPath(env?: Record<string, string | undefined>): string;
|
|
49
|
+
|
|
50
|
+
export function initPortfolioQueueStore(dbPath?: string): PortfolioQueueStore;
|
|
51
|
+
|
|
52
|
+
export function enqueue(item: EnqueueItem): QueueEntry;
|
|
53
|
+
|
|
54
|
+
export function dequeueNext(): QueueEntry | null;
|
|
55
|
+
|
|
56
|
+
export function listQueue(repoFullName?: string | null): QueueEntry[];
|
|
57
|
+
|
|
58
|
+
export function markDone(repoFullName: string, identifier: string, apiBaseUrl?: string): QueueEntry | null;
|
|
59
|
+
|
|
60
|
+
export function markFailed(repoFullName: string, identifier: string, apiBaseUrl?: string): QueueEntry | null;
|
|
61
|
+
|
|
62
|
+
export function closeDefaultPortfolioQueueStore(): void;
|
|
@@ -0,0 +1,351 @@
|
|
|
1
|
+
import { DEFAULT_FORGE_CONFIG } from "./forge-config.js";
|
|
2
|
+
import { normalizeLocalStoreDbPath, openLocalStoreDb, resolveLocalStoreDbPath } from "./local-store.js";
|
|
3
|
+
import { applySchemaMigrations } from "./schema-version.js";
|
|
4
|
+
|
|
5
|
+
// The miner's local portfolio/queue store (#2292): a 100% client-side, prioritized backlog of candidate work
|
|
6
|
+
// items across every repo the miner has been pointed at ("what should I look at next, across everything I'm
|
|
7
|
+
// tracking"). The database only lives on this machine; this module never uploads, syncs, or phones home with its
|
|
8
|
+
// contents. The `priority` field is a PLACEHOLDER numeric input in this foundation phase — later phases populate
|
|
9
|
+
// it from the extracted reward-risk/scoring modules in `gittensory-engine`; it is not invented here.
|
|
10
|
+
|
|
11
|
+
export const QUEUE_STATUSES = Object.freeze(["queued", "in_progress", "done"]);
|
|
12
|
+
|
|
13
|
+
const defaultDbFileName = "portfolio-queue.sqlite3";
|
|
14
|
+
let defaultPortfolioQueueStore = null;
|
|
15
|
+
|
|
16
|
+
export function resolvePortfolioQueueDbPath(env = process.env) {
|
|
17
|
+
return resolveLocalStoreDbPath(defaultDbFileName, "GITTENSORY_MINER_PORTFOLIO_QUEUE_DB", env);
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
function normalizeDbPath(dbPath) {
|
|
21
|
+
return normalizeLocalStoreDbPath(dbPath, resolvePortfolioQueueDbPath(), "invalid_portfolio_queue_db_path");
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function normalizeRepoFullName(repoFullName) {
|
|
25
|
+
if (typeof repoFullName !== "string") throw new Error("invalid_repo_full_name");
|
|
26
|
+
const trimmed = repoFullName.trim();
|
|
27
|
+
const [owner, repo, extra] = trimmed.split("/");
|
|
28
|
+
if (!owner || !repo || extra !== undefined) throw new Error("invalid_repo_full_name");
|
|
29
|
+
return `${owner}/${repo}`;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function normalizeIdentifier(identifier) {
|
|
33
|
+
if (typeof identifier !== "string") throw new Error("invalid_identifier");
|
|
34
|
+
const trimmed = identifier.trim();
|
|
35
|
+
if (!trimmed) throw new Error("invalid_identifier");
|
|
36
|
+
return trimmed;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** Priority is a placeholder numeric input; an omitted priority defaults to 0, a non-finite or negative one is rejected. */
|
|
40
|
+
function normalizePriority(priority) {
|
|
41
|
+
if (priority === undefined || priority === null) return 0;
|
|
42
|
+
if (typeof priority !== "number" || !Number.isFinite(priority) || priority < 0) {
|
|
43
|
+
throw new Error("invalid_priority");
|
|
44
|
+
}
|
|
45
|
+
return priority;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** Optional forge host, scoping rows so two hosts serving the same owner/repo name never collide (#5563).
|
|
49
|
+
* Omitted/nullish → the github.com default, so every pre-existing single-forge caller is unaffected. */
|
|
50
|
+
function normalizeApiBaseUrl(apiBaseUrl) {
|
|
51
|
+
if (apiBaseUrl === undefined || apiBaseUrl === null) return DEFAULT_FORGE_CONFIG.apiBaseUrl;
|
|
52
|
+
if (typeof apiBaseUrl !== "string" || !apiBaseUrl.trim()) throw new Error("invalid_api_base_url");
|
|
53
|
+
return apiBaseUrl.trim();
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function rowToEntry(row) {
|
|
57
|
+
return {
|
|
58
|
+
apiBaseUrl: row.api_base_url,
|
|
59
|
+
repoFullName: row.repo_full_name,
|
|
60
|
+
identifier: row.identifier,
|
|
61
|
+
priority: row.priority,
|
|
62
|
+
status: row.status,
|
|
63
|
+
enqueuedAt: row.enqueued_at,
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** Lease-annotated projection of an in-flight row (adds `leasedAt`), consumed by the expiry sweep. Kept separate
|
|
68
|
+
* from `rowToEntry` so the base entry shape every existing caller relies on is unchanged. */
|
|
69
|
+
function rowToLeaseEntry(row) {
|
|
70
|
+
return {
|
|
71
|
+
apiBaseUrl: row.api_base_url,
|
|
72
|
+
repoFullName: row.repo_full_name,
|
|
73
|
+
identifier: row.identifier,
|
|
74
|
+
status: row.status,
|
|
75
|
+
leasedAt: row.leased_at ?? null,
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Opens the local portfolio/queue store, creating the table on first use. Rows are ordered highest-priority-first
|
|
81
|
+
* with an insertion-order tie-break: `priority DESC, enqueued_at ASC, rowid ASC` — the implicit `rowid` guarantees
|
|
82
|
+
* FIFO order even when two items share a priority AND an `enqueued_at` timestamp. (#2292)
|
|
83
|
+
*/
|
|
84
|
+
export function initPortfolioQueueStore(dbPath = resolvePortfolioQueueDbPath()) {
|
|
85
|
+
const resolvedPath = normalizeDbPath(dbPath);
|
|
86
|
+
// openLocalStoreDb skips mkdir/chmod for the special in-memory path (':memory:'), which has no file on disk.
|
|
87
|
+
const db = openLocalStoreDb(resolvedPath);
|
|
88
|
+
db.exec(`
|
|
89
|
+
CREATE TABLE IF NOT EXISTS miner_portfolio_queue (
|
|
90
|
+
repo_full_name TEXT NOT NULL,
|
|
91
|
+
identifier TEXT NOT NULL,
|
|
92
|
+
priority REAL NOT NULL DEFAULT 0,
|
|
93
|
+
status TEXT NOT NULL DEFAULT 'queued' CHECK (status IN ('queued', 'in_progress', 'done')),
|
|
94
|
+
enqueued_at TEXT NOT NULL,
|
|
95
|
+
leased_at TEXT,
|
|
96
|
+
PRIMARY KEY (repo_full_name, identifier)
|
|
97
|
+
)
|
|
98
|
+
`);
|
|
99
|
+
// `leased_at` records when an item was flipped to 'in_progress', so a crashed/killed process's stuck lease can be
|
|
100
|
+
// swept back to 'queued' by age (see portfolio-queue-expiry.js) instead of stranding the item forever — the same
|
|
101
|
+
// recovery the claim-ledger and worktree-allocator stores already provide for their own tables (#4827). Additive
|
|
102
|
+
// migration for stores created before this column: CREATE TABLE IF NOT EXISTS never adds a column to a pre-existing
|
|
103
|
+
// table, so add it idempotently. Expressed as the store's first schema migration (#4832): the baseline table is
|
|
104
|
+
// version 1; migration 1→2 adds `leased_at`. The migration stays defensive (checks table_info) so a version-0
|
|
105
|
+
// file that already ran the pre-convention ad-hoc ALTER is not re-altered into a duplicate-column error.
|
|
106
|
+
//
|
|
107
|
+
// v2 -> v3 (#5563): rebuild PRIMARY KEY (repo_full_name, identifier) into PRIMARY KEY (api_base_url,
|
|
108
|
+
// repo_full_name, identifier) -- two forge hosts serving a same-named owner/repo must not collide in this
|
|
109
|
+
// queue. SQLite cannot ALTER a PRIMARY KEY in place, so this rebuilds the table: create the new shape, copy
|
|
110
|
+
// every existing row with the pre-#4784 implicit single-forge default backfilled, drop the old table, rename
|
|
111
|
+
// the new one in.
|
|
112
|
+
applySchemaMigrations(db, [
|
|
113
|
+
(migrationDb) => {
|
|
114
|
+
const hasLeasedAtColumn = migrationDb
|
|
115
|
+
.prepare("PRAGMA table_info(miner_portfolio_queue)")
|
|
116
|
+
.all()
|
|
117
|
+
.some((column) => column.name === "leased_at");
|
|
118
|
+
if (!hasLeasedAtColumn) migrationDb.exec("ALTER TABLE miner_portfolio_queue ADD COLUMN leased_at TEXT");
|
|
119
|
+
},
|
|
120
|
+
(migrationDb) => {
|
|
121
|
+
migrationDb.exec(`
|
|
122
|
+
CREATE TABLE miner_portfolio_queue_v3 (
|
|
123
|
+
api_base_url TEXT NOT NULL,
|
|
124
|
+
repo_full_name TEXT NOT NULL,
|
|
125
|
+
identifier TEXT NOT NULL,
|
|
126
|
+
priority REAL NOT NULL DEFAULT 0,
|
|
127
|
+
status TEXT NOT NULL DEFAULT 'queued' CHECK (status IN ('queued', 'in_progress', 'done')),
|
|
128
|
+
enqueued_at TEXT NOT NULL,
|
|
129
|
+
leased_at TEXT,
|
|
130
|
+
PRIMARY KEY (api_base_url, repo_full_name, identifier)
|
|
131
|
+
)
|
|
132
|
+
`);
|
|
133
|
+
// ORDER BY rowid preserves the old table's FIFO insertion order in the new table's freshly-assigned rowids
|
|
134
|
+
// (the composite PRIMARY KEY above is not itself the rowid), so this rebuild doesn't reshuffle queue order.
|
|
135
|
+
// OR IGNORE: a row this store's own read path already treats as unusable garbage (an unrecognized
|
|
136
|
+
// `status`, e.g. from a hand-edited or otherwise corrupted file) would violate the CHECK constraint above
|
|
137
|
+
// and abort the whole migration. Skipping it here is consistent with that same fail-closed posture, rather
|
|
138
|
+
// than turning one bad row into a permanently unmigratable file.
|
|
139
|
+
migrationDb
|
|
140
|
+
.prepare(
|
|
141
|
+
`INSERT OR IGNORE INTO miner_portfolio_queue_v3
|
|
142
|
+
(api_base_url, repo_full_name, identifier, priority, status, enqueued_at, leased_at)
|
|
143
|
+
SELECT ?, repo_full_name, identifier, priority, status, enqueued_at, leased_at
|
|
144
|
+
FROM miner_portfolio_queue ORDER BY rowid`,
|
|
145
|
+
)
|
|
146
|
+
.run(DEFAULT_FORGE_CONFIG.apiBaseUrl);
|
|
147
|
+
migrationDb.exec("DROP TABLE miner_portfolio_queue");
|
|
148
|
+
migrationDb.exec("ALTER TABLE miner_portfolio_queue_v3 RENAME TO miner_portfolio_queue");
|
|
149
|
+
},
|
|
150
|
+
]);
|
|
151
|
+
|
|
152
|
+
// `rowid` is a stable, unique key assigned once at first insert (re-enqueue updates in place, never re-inserts),
|
|
153
|
+
// so it is a deterministic total-order tie-break: two items sharing a priority AND an `enqueued_at` timestamp
|
|
154
|
+
// still order by insertion.
|
|
155
|
+
const ORDER = "ORDER BY priority DESC, enqueued_at ASC, rowid ASC";
|
|
156
|
+
// Re-enqueueing an already-tracked item re-activates it IN PLACE: refresh its (placeholder) priority and reset it
|
|
157
|
+
// to 'queued', but KEEP the original `enqueued_at` and `rowid` so it holds its existing FIFO position rather than
|
|
158
|
+
// jumping the queue. (Restamping `enqueued_at` would be inconsistent — the fixed `rowid` still pins the old
|
|
159
|
+
// position whenever timestamps collide — so position is deliberately preserved instead.)
|
|
160
|
+
const enqueueStatement = db.prepare(`
|
|
161
|
+
INSERT INTO miner_portfolio_queue (api_base_url, repo_full_name, identifier, priority, status, enqueued_at)
|
|
162
|
+
VALUES (?, ?, ?, ?, 'queued', ?)
|
|
163
|
+
ON CONFLICT(api_base_url, repo_full_name, identifier) DO UPDATE SET
|
|
164
|
+
priority = excluded.priority,
|
|
165
|
+
status = 'queued'
|
|
166
|
+
WHERE miner_portfolio_queue.status <> 'in_progress'
|
|
167
|
+
`);
|
|
168
|
+
const getStatement = db.prepare(
|
|
169
|
+
"SELECT * FROM miner_portfolio_queue WHERE api_base_url = ? AND repo_full_name = ? AND identifier = ?",
|
|
170
|
+
);
|
|
171
|
+
// Claim the highest-priority queued item ATOMICALLY: one UPDATE selects the ordered top row in a subquery and
|
|
172
|
+
// flips it to 'in_progress', RETURNING it — so two processes sharing the file can't both claim the same row (a
|
|
173
|
+
// separate SELECT-then-UPDATE would race). Deliberately global (no api_base_url filter): the queue is a single
|
|
174
|
+
// cross-host priority ordering, not a per-host one.
|
|
175
|
+
// Claiming stamps `leased_at` with the caller-supplied claim time; leaving 'in_progress' (done/failed/reclaim)
|
|
176
|
+
// clears it back to NULL so only genuinely in-flight rows carry a lease.
|
|
177
|
+
const dequeueStatement = db.prepare(`
|
|
178
|
+
UPDATE miner_portfolio_queue SET status = 'in_progress', leased_at = ?
|
|
179
|
+
WHERE rowid = (
|
|
180
|
+
SELECT rowid FROM miner_portfolio_queue WHERE status = 'queued' ${ORDER} LIMIT 1
|
|
181
|
+
)
|
|
182
|
+
RETURNING *
|
|
183
|
+
`);
|
|
184
|
+
// RETURNING (rather than a separate post-UPDATE SELECT) makes the "nothing to mark done" case observable
|
|
185
|
+
// directly from one atomic statement.
|
|
186
|
+
const markDoneStatement = db.prepare(`
|
|
187
|
+
UPDATE miner_portfolio_queue SET status = 'done', leased_at = NULL
|
|
188
|
+
WHERE api_base_url = ? AND repo_full_name = ? AND identifier = ? AND status <> 'done'
|
|
189
|
+
RETURNING *
|
|
190
|
+
`);
|
|
191
|
+
const markFailedStatement = db.prepare(`
|
|
192
|
+
UPDATE miner_portfolio_queue SET status = 'queued', leased_at = NULL
|
|
193
|
+
WHERE api_base_url = ? AND repo_full_name = ? AND identifier = ? AND status = 'in_progress'
|
|
194
|
+
RETURNING *
|
|
195
|
+
`);
|
|
196
|
+
const listAllStatement = db.prepare(`SELECT * FROM miner_portfolio_queue ${ORDER}`);
|
|
197
|
+
const listRepoStatement = db.prepare(
|
|
198
|
+
`SELECT * FROM miner_portfolio_queue WHERE repo_full_name = ? ${ORDER}`,
|
|
199
|
+
);
|
|
200
|
+
const listActiveStatement = db.prepare(
|
|
201
|
+
`SELECT * FROM miner_portfolio_queue WHERE status IN ('queued', 'in_progress') ${ORDER}`,
|
|
202
|
+
);
|
|
203
|
+
const listInProgressStatement = db.prepare(
|
|
204
|
+
`SELECT * FROM miner_portfolio_queue WHERE status = 'in_progress' ${ORDER}`,
|
|
205
|
+
);
|
|
206
|
+
const reclaimStatement = db.prepare(`
|
|
207
|
+
UPDATE miner_portfolio_queue SET status = 'queued', leased_at = NULL
|
|
208
|
+
WHERE api_base_url = ? AND repo_full_name = ? AND identifier = ? AND status = 'in_progress'
|
|
209
|
+
RETURNING *
|
|
210
|
+
`);
|
|
211
|
+
// Requeue only ever targets a COMPLETED ('done') row — an in-flight item is released via reclaimStatement, and
|
|
212
|
+
// an already-'queued' item is a no-op — so a caller's manual requeue can never disturb an active claim. The
|
|
213
|
+
// row keeps its rowid/enqueued_at, so it re-enters the queue at its original FIFO position, not the back.
|
|
214
|
+
const requeueStatement = db.prepare(`
|
|
215
|
+
UPDATE miner_portfolio_queue SET status = 'queued', leased_at = NULL
|
|
216
|
+
WHERE api_base_url = ? AND repo_full_name = ? AND identifier = ? AND status = 'done'
|
|
217
|
+
RETURNING *
|
|
218
|
+
`);
|
|
219
|
+
const claimTargetStatement = db.prepare(`
|
|
220
|
+
UPDATE miner_portfolio_queue SET status = 'in_progress', leased_at = ?
|
|
221
|
+
WHERE api_base_url = ? AND repo_full_name = ? AND identifier = ? AND status = 'queued'
|
|
222
|
+
RETURNING *
|
|
223
|
+
`);
|
|
224
|
+
|
|
225
|
+
return {
|
|
226
|
+
dbPath: resolvedPath,
|
|
227
|
+
enqueue(item) {
|
|
228
|
+
const apiBaseUrl = normalizeApiBaseUrl(item?.apiBaseUrl);
|
|
229
|
+
const repoFullName = normalizeRepoFullName(item?.repoFullName);
|
|
230
|
+
const identifier = normalizeIdentifier(item?.identifier);
|
|
231
|
+
const priority = normalizePriority(item?.priority);
|
|
232
|
+
const enqueuedAt = new Date().toISOString();
|
|
233
|
+
enqueueStatement.run(apiBaseUrl, repoFullName, identifier, priority, enqueuedAt);
|
|
234
|
+
return rowToEntry(getStatement.get(apiBaseUrl, repoFullName, identifier));
|
|
235
|
+
},
|
|
236
|
+
dequeueNext() {
|
|
237
|
+
const row = dequeueStatement.get(new Date().toISOString());
|
|
238
|
+
return row ? rowToEntry(row) : null;
|
|
239
|
+
},
|
|
240
|
+
/** In-flight ('in_progress') rows with their `leasedAt` claim time, for the expiry sweep (#4827). */
|
|
241
|
+
listInProgress() {
|
|
242
|
+
return listInProgressStatement.all().map(rowToLeaseEntry);
|
|
243
|
+
},
|
|
244
|
+
/** Reclaim a single stuck in-flight item back to 'queued' (clearing its lease), returning it — or null if it is
|
|
245
|
+
* no longer 'in_progress' (already finished/reclaimed by another sweep). The sweep target of #4827. */
|
|
246
|
+
reclaimStuckItem(repoFullName, identifier, apiBaseUrl) {
|
|
247
|
+
const row = reclaimStatement.get(
|
|
248
|
+
normalizeApiBaseUrl(apiBaseUrl),
|
|
249
|
+
normalizeRepoFullName(repoFullName),
|
|
250
|
+
normalizeIdentifier(identifier),
|
|
251
|
+
);
|
|
252
|
+
return row ? rowToEntry(row) : null;
|
|
253
|
+
},
|
|
254
|
+
/** Requeue a COMPLETED ('done') item back to 'queued' so it is picked up again, keeping its FIFO position
|
|
255
|
+
* (rowid/enqueued_at unchanged). Returns the entry, or null when there is no 'done' item to requeue — i.e.
|
|
256
|
+
* it is already 'queued', is currently 'in_progress' (release it via {@link reclaimStuckItem} instead), or
|
|
257
|
+
* does not exist. The manual counterpart to {@link reclaimStuckItem} for the queue CLI's escape hatch (#4828). */
|
|
258
|
+
requeueItem(repoFullName, identifier, apiBaseUrl) {
|
|
259
|
+
const row = requeueStatement.get(
|
|
260
|
+
normalizeApiBaseUrl(apiBaseUrl),
|
|
261
|
+
normalizeRepoFullName(repoFullName),
|
|
262
|
+
normalizeIdentifier(identifier),
|
|
263
|
+
);
|
|
264
|
+
return row ? rowToEntry(row) : null;
|
|
265
|
+
},
|
|
266
|
+
listQueue(repoFullName) {
|
|
267
|
+
const rows = repoFullName === undefined || repoFullName === null
|
|
268
|
+
? listAllStatement.all()
|
|
269
|
+
: listRepoStatement.all(normalizeRepoFullName(repoFullName));
|
|
270
|
+
return rows.map(rowToEntry);
|
|
271
|
+
},
|
|
272
|
+
markDone(repoFullName, identifier, apiBaseUrl) {
|
|
273
|
+
const row = markDoneStatement.get(
|
|
274
|
+
normalizeApiBaseUrl(apiBaseUrl),
|
|
275
|
+
normalizeRepoFullName(repoFullName),
|
|
276
|
+
normalizeIdentifier(identifier),
|
|
277
|
+
);
|
|
278
|
+
return row ? rowToEntry(row) : null;
|
|
279
|
+
},
|
|
280
|
+
/** Release an in-flight item back to `queued` when a run halts (#2347). */
|
|
281
|
+
markFailed(repoFullName, identifier, apiBaseUrl) {
|
|
282
|
+
const row = markFailedStatement.get(
|
|
283
|
+
normalizeApiBaseUrl(apiBaseUrl),
|
|
284
|
+
normalizeRepoFullName(repoFullName),
|
|
285
|
+
normalizeIdentifier(identifier),
|
|
286
|
+
);
|
|
287
|
+
return row ? rowToEntry(row) : null;
|
|
288
|
+
},
|
|
289
|
+
/**
|
|
290
|
+
* Transactional caps-aware batch claim hook used by portfolio-queue-manager.js: re-read active rows under an
|
|
291
|
+
* exclusive lock, let the caller pick targets, then atomically flip each still-queued row to `in_progress`.
|
|
292
|
+
*/
|
|
293
|
+
batchClaim(selectFn) {
|
|
294
|
+
if (typeof selectFn !== "function") throw new Error("invalid_batch_claim_selector");
|
|
295
|
+
db.exec("BEGIN IMMEDIATE");
|
|
296
|
+
try {
|
|
297
|
+
const entries = listActiveStatement.all().map(rowToEntry);
|
|
298
|
+
const targets = selectFn(entries);
|
|
299
|
+
if (!Array.isArray(targets)) throw new Error("invalid_batch_claim_selection");
|
|
300
|
+
const leasedAt = new Date().toISOString();
|
|
301
|
+
const claimed = [];
|
|
302
|
+
for (const target of targets) {
|
|
303
|
+
const apiBaseUrl = normalizeApiBaseUrl(target?.apiBaseUrl);
|
|
304
|
+
const repoFullName = normalizeRepoFullName(target?.repoFullName);
|
|
305
|
+
const identifier = normalizeIdentifier(target?.identifier);
|
|
306
|
+
const row = claimTargetStatement.get(leasedAt, apiBaseUrl, repoFullName, identifier);
|
|
307
|
+
if (row) claimed.push(rowToEntry(row));
|
|
308
|
+
}
|
|
309
|
+
db.exec("COMMIT");
|
|
310
|
+
return claimed;
|
|
311
|
+
} catch (error) {
|
|
312
|
+
db.exec("ROLLBACK");
|
|
313
|
+
throw error;
|
|
314
|
+
}
|
|
315
|
+
},
|
|
316
|
+
close() {
|
|
317
|
+
db.close();
|
|
318
|
+
},
|
|
319
|
+
};
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
function getDefaultPortfolioQueueStore() {
|
|
323
|
+
defaultPortfolioQueueStore ??= initPortfolioQueueStore();
|
|
324
|
+
return defaultPortfolioQueueStore;
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
export function enqueue(item) {
|
|
328
|
+
return getDefaultPortfolioQueueStore().enqueue(item);
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
export function dequeueNext() {
|
|
332
|
+
return getDefaultPortfolioQueueStore().dequeueNext();
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
export function listQueue(repoFullName) {
|
|
336
|
+
return getDefaultPortfolioQueueStore().listQueue(repoFullName);
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
export function markDone(repoFullName, identifier, apiBaseUrl) {
|
|
340
|
+
return getDefaultPortfolioQueueStore().markDone(repoFullName, identifier, apiBaseUrl);
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
export function markFailed(repoFullName, identifier, apiBaseUrl) {
|
|
344
|
+
return getDefaultPortfolioQueueStore().markFailed(repoFullName, identifier, apiBaseUrl);
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
export function closeDefaultPortfolioQueueStore() {
|
|
348
|
+
if (!defaultPortfolioQueueStore) return;
|
|
349
|
+
defaultPortfolioQueueStore.close();
|
|
350
|
+
defaultPortfolioQueueStore = null;
|
|
351
|
+
}
|