@powerhousedao/registry 6.2.3-dev.39 → 6.2.3-dev.40

package/dist/cli.d.mts

@@ -10,6 +10,7 @@ export declare const registryCommand: Partial<import("cmd-ts/dist/cjs/argparser.
     s3SecretAccessKey: string | undefined;
     s3KeyPrefix: string | undefined;
     s3ForcePathStyle: boolean;
+    s3MaxWritesPerSecond: number | undefined;
     databaseUrl: string | undefined;
     listenDatabaseUrl: string | undefined;
     webhooks: string | undefined;
@@ -40,6 +41,7 @@ export declare const registryCommand: Partial<import("cmd-ts/dist/cjs/argparser.
   s3SecretAccessKey: string | undefined;
   s3KeyPrefix: string | undefined;
   s3ForcePathStyle: boolean;
+  s3MaxWritesPerSecond: number | undefined;
   databaseUrl: string | undefined;
   listenDatabaseUrl: string | undefined;
   webhooks: string | undefined;
@@ -73,6 +75,7 @@ export declare const workerCommand: Partial<import("cmd-ts/dist/cjs/argparser.js
     s3SecretAccessKey: string | undefined;
     s3KeyPrefix: string | undefined;
     s3ForcePathStyle: boolean;
+    s3MaxWritesPerSecond: number | undefined;
     databaseUrl: string | undefined;
     listenDatabaseUrl: string | undefined;
     webhooks: string | undefined;
@@ -92,6 +95,7 @@ export declare const workerCommand: Partial<import("cmd-ts/dist/cjs/argparser.js
   s3SecretAccessKey: string | undefined;
   s3KeyPrefix: string | undefined;
   s3ForcePathStyle: boolean;
+  s3MaxWritesPerSecond: number | undefined;
   databaseUrl: string | undefined;
   listenDatabaseUrl: string | undefined;
   webhooks: string | undefined;
@@ -127,6 +131,7 @@ export declare const importVerdaccioStateCommand: Partial<import("cmd-ts/dist/cj
     s3SecretAccessKey: string | undefined;
     s3KeyPrefix: string | undefined;
     s3ForcePathStyle: boolean;
+    s3MaxWritesPerSecond: number | undefined;
     databaseUrl: string | undefined;
     listenDatabaseUrl: string | undefined;
     webhooks: string | undefined;
@@ -141,6 +146,7 @@ export declare const importVerdaccioStateCommand: Partial<import("cmd-ts/dist/cj
   s3SecretAccessKey: string | undefined;
   s3KeyPrefix: string | undefined;
   s3ForcePathStyle: boolean;
+  s3MaxWritesPerSecond: number | undefined;
   databaseUrl: string | undefined;
   listenDatabaseUrl: string | undefined;
   webhooks: string | undefined;

package/dist/cli.d.mts.map

@@ -1 +1 @@
-{"version":3,"file":"cli.d.mts","names":[],"sources":["../cli.ts"],"mappings":";qBA2Ia,iBAAe,+CAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IA2I1B,6CAAA,iDAAA,mDAAA;qBAEW,eAAa,+CAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAyCxB,6CAAA,iDAAA,mDAAA;qBAEW,gBAAc,+CAAA;;;;;;;;;;;;IAkBzB,6CAAA,iDAAA,mDAAA;qBAEW,6BAA2B,+CAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAatC,6CAAA,iDAAA,mDAAA"}
+{"version":3,"file":"cli.d.mts","names":[],"sources":["../cli.ts"],"mappings":";qBAkJa,iBAAe,+CAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IA2I1B,6CAAA,iDAAA,mDAAA;qBAEW,eAAa,+CAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAyCxB,6CAAA,iDAAA,mDAAA;qBAEW,gBAAc,+CAAA;;;;;;;;;;;;IAkBzB,6CAAA,iDAAA,mDAAA;qBAEW,6BAA2B,+CAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAatC,6CAAA,iDAAA,mDAAA"}

package/dist/cli.mjs

@@ -1882,9 +1882,27 @@ async function forgetWith(query, name) {
 	await query("DELETE FROM verdaccio_manifests WHERE name = $1", [name]);
 }
 //#endregion
+//#region src/pace.ts
+/** Spaces calls to at most `perSecond`, in the order they ask. */
+function pacer(perSecond) {
+	const gap = 1e3 / perSecond;
+	let next = 0;
+	return () => {
+		const now = Date.now();
+		const at = Math.max(now, next);
+		next = at + gap;
+		return at > now ? new Promise((resolve) => setTimeout(resolve, at - now)) : Promise.resolve();
+	};
+}
+//#endregion
 //#region src/artifacts.ts
+const BUFFERED_UPLOAD_BYTES = 4194304;
+const DEFAULT_WRITES_PER_SECOND = 150;
 function createS3ArtifactStore(config) {
+	const paceWrite = pacer(config.maxWritesPerSecond ?? DEFAULT_WRITES_PER_SECOND);
 	const s3 = new S3Client({
+		retryMode: "adaptive",
+		maxAttempts: 5,
 		endpoint: config.endpoint,
 		region: config.region,
 		forcePathStyle: config.s3ForcePathStyle ?? true,
@@ -1910,6 +1928,7 @@ function createS3ArtifactStore(config) {
 	const root = `${config.keyPrefix ?? ""}artifacts/`;
 	return {
 		async put(key, body, contentType) {
+			await paceWrite();
 			await s3.send(new PutObjectCommand({
 				Bucket: config.bucket,
 				Key: root + key,
@@ -1919,9 +1938,22 @@ function createS3ArtifactStore(config) {
 		},
 		async putFile(key, file, contentType) {
 			const { size } = await fs.promises.stat(file);
+			if (size <= BUFFERED_UPLOAD_BYTES) {
+				const body = await fs.promises.readFile(file);
+				await paceWrite();
+				await s3.send(new PutObjectCommand({
+					Bucket: config.bucket,
+					Key: root + key,
+					Body: body,
+					ContentLength: size,
+					ContentType: contentType
+				}));
+				return;
+			}
 			const body = fs.createReadStream(file);
 			body.on("error", () => void 0);
 			try {
+				await paceWrite();
 				await s3.send(new PutObjectCommand({
 					Bucket: config.bucket,
 					Key: root + key,
@@ -1959,10 +1991,13 @@ function createS3ArtifactStore(config) {
 					ContinuationToken: token
 				}));
 				const keys = (listed.Contents ?? []).flatMap((o) => o.Key ? [{ Key: o.Key }] : []);
-				if (keys.length > 0) await s3.send(new DeleteObjectsCommand({
-					Bucket: config.bucket,
-					Delete: { Objects: keys }
-				}));
+				if (keys.length > 0) {
+					await paceWrite();
+					await s3.send(new DeleteObjectsCommand({
+						Bucket: config.bucket,
+						Delete: { Objects: keys }
+					}));
+				}
 				token = listed.IsTruncated ? listed.NextContinuationToken : void 0;
 			} while (token);
 		}
@@ -2406,6 +2441,12 @@ const MIGRATIONS = [
 		name: "reconcile by revision",
 		transaction: false,
 		statements: ["ALTER TABLE registry_packages ADD COLUMN IF NOT EXISTS manifest_rev text", ...indexConcurrently("registry_versions_pending", "ON registry_versions (updated_at) WHERE status = 'pending'")]
+	},
+	{
+		id: 7,
+		name: "permanent version failures",
+		transaction: false,
+		statements: ["ALTER TABLE registry_versions ADD COLUMN IF NOT EXISTS permanent boolean NOT NULL DEFAULT false"]
 	}
 ];
 const MIGRATIONS_TABLE = `CREATE TABLE IF NOT EXISTS registry_migrations (
@@ -3036,9 +3077,11 @@ function publisherOf$1(job) {
 async function processVersion(ctx, job, finish) {
 	const pkg = job.package;
 	const version = job.version;
-	const local = await publishedHere(ctx.db, pkg) ?? job.payload.local === true;
+	const published = await publishedHere(ctx.db, pkg);
+	const local = published ?? job.payload.local === true;
 	const manifestRev = await indexedRevision(ctx.db, pkg);
 	const packument = await fetchPackument(ctx.registryUrl, pkg);
+	if (!packument && published) throw new Error(`metadata for published ${pkg} returned 404`);
 	if (!packument?.versions?.[version]) {
 		await ctx.db.transaction(async (tx) => {
 			await removeVersionRows(tx, pkg, [version]);
@@ -3098,14 +3141,16 @@ async function processVersion(ctx, job, finish) {
 	return true;
 }
 /** Records a version that can't be processed. */
-async function failVersion(ctx, job, error) {
-	await ctx.db.query(`INSERT INTO registry_versions (package, version, status, error)
-     VALUES ($1, $2, 'failed', $3)
+async function failVersion(ctx, job, error, permanent = false) {
+	await ctx.db.query(`INSERT INTO registry_versions (package, version, status, error, permanent)
+     VALUES ($1, $2, 'failed', $3, $4)
      ON CONFLICT (package, version) DO UPDATE SET
-       status = 'failed', error = EXCLUDED.error, updated_at = now()`, [
+       status = 'failed', error = EXCLUDED.error,
+       permanent = EXCLUDED.permanent, updated_at = now()`, [
 		job.package,
 		job.version,
-		error
+		error,
+		permanent
 	]);
 	await ctx.events.publish({
 		type: "version-failed",
@@ -3163,6 +3208,7 @@ const CACHE_TTL_MS = 3e4;
 const POLL_MS = 1e3;
 const ON_DEMAND_TIMEOUT_MS = 6e4;
 const VERSION_CACHE_MAX = 5e3;
+const FAILED_RETRY_AFTER = "15 minutes";
 function pieceColumns(table) {
 	return `${table}.name, ${table}.version, ${table}.package,
     ${table}.display_name AS "displayName", ${table}.description,
@@ -3330,8 +3376,14 @@ var Catalog = class {
 		const cached = this.#versions.get(key);
 		if (this.#fresh(cached) && cached.value?.status === "ready") return cached.value;
 		const row = (await this.#db.query(`SELECT package, version, status, manifest,
-              package_json_version AS "packageJsonVersion", files, error
-         FROM registry_versions WHERE package = $1 AND version = $2`, [pkg, version])).rows[0] ?? null;
+              package_json_version AS "packageJsonVersion", files, error,
+              (status = 'failed' AND NOT permanent
+               AND updated_at < now() - $3::interval) AS retryable
+         FROM registry_versions WHERE package = $1 AND version = $2`, [
+			pkg,
+			version,
+			FAILED_RETRY_AFTER
+		])).rows[0] ?? null;
 		if (row?.status === "ready") {
 			this.#versions.delete(key);
 			this.#versions.set(key, {
@@ -3357,11 +3409,20 @@ var Catalog = class {
 			kind: "ready",
 			row
 		};
-		if (row?.status === "failed") return {
+		if (row?.status === "failed" && !row.retryable) return {
 			kind: "failed",
 			error: row.error ?? ""
 		};
-		if (!row) {
+		if (row?.status === "failed") {
+			const local = (await this.packageRow(pkg))?.local === true;
+			await this.#db.transaction(async (tx) => {
+				if ((await tx.query(`UPDATE registry_versions SET status = 'pending', error = NULL, updated_at = now()
+            WHERE package = $1 AND version = $2 AND status = 'failed'
+            RETURNING version`, [pkg, version])).rows.length === 0) return;
+				await enqueue(tx, "process", pkg, version, {}, local ? 0 : 5);
+			});
+			await wakeWorkers(this.#db);
+		} else if (!row) {
 			if (!(await fetchPackument(this.#registryUrl(), pkg))?.versions?.[version]) return { kind: "missing" };
 			const local = (await this.packageRow(pkg))?.local === true;
 			await this.#db.transaction(async (tx) => {
@@ -3623,7 +3684,9 @@ const SECONDS_BUCKETS = [
 	1,
 	2.5,
 	5,
-	10
+	10,
+	30,
+	60
 ];
 const JOB_BUCKETS = [
 	.05,
@@ -3637,9 +3700,12 @@ const JOB_BUCKETS = [
 	30,
 	60,
 	120,
-	300
+	300,
+	600,
+	1800
 ];
 const QUEUE_CACHE_MS = 5e3;
+const CATALOG_CACHE_MS = 3e4;
 /** Event-loop delay, memory and CPU of this process. */
 function processMetrics(metrics) {
 	const loop = monitorEventLoopDelay({ resolution: 10 });
@@ -3734,9 +3800,65 @@ function databaseMetrics(metrics, db) {
 	metrics.gauge("registry_jobs_oldest_due_seconds", "Age of the oldest unclaimed due job, by priority", byPriority("oldest_due"));
 	metrics.gauge("registry_listen_connected", "1 while the LISTEN connection is up", () => [{ value: db.listening() ? 1 : 0 }]);
 }
+/**
+* Package and version counts. They're the same from every process, so only
+* one that runs jobs registers them.
+*/
+function catalogMetrics(metrics, db) {
+	let catalogCached;
+	const catalog = () => {
+		if (!catalogCached || Date.now() - catalogCached.at > CATALOG_CACHE_MS) {
+			const row = db.query(`SELECT
+             (SELECT count(*) FROM registry_packages WHERE local)::int AS local,
+             (SELECT count(*) FROM registry_packages
+               WHERE local AND listed_manifest IS NOT NULL)::int AS listed,
+             count(*) FILTER (WHERE status = 'ready')::int AS ready,
+             count(*) FILTER (WHERE status = 'pending')::int AS pending,
+             count(*) FILTER (WHERE status = 'failed' AND NOT permanent)::int AS failed,
+             count(*) FILTER (WHERE status = 'failed' AND permanent)::int AS failed_permanent
+             FROM registry_versions`).then((r) => r.rows[0]);
+			catalogCached = {
+				at: Date.now(),
+				row
+			};
+			row.catch(() => catalogCached = void 0);
+		}
+		return catalogCached.row;
+	};
+	metrics.gauge("registry_packages", "Packages published here (local), and those /packages lists (listed)", async () => {
+		const row = await catalog();
+		return ["local", "listed"].map((state) => ({
+			labels: { state },
+			value: row[state]
+		}));
+	});
+	metrics.gauge("registry_versions", "Processed versions, by status; failed_permanent is never retried", async () => {
+		const row = await catalog();
+		return [
+			"ready",
+			"pending",
+			"failed",
+			"failed_permanent"
+		].map((status) => ({
+			labels: { status },
+			value: row[status]
+		}));
+	});
+}
+/** A bounded label for why a job failed. */
+function errorReason(err, permanent = false) {
+	const name = err.name ?? "";
+	const message = err instanceof Error ? err.message : String(err);
+	if (name === "SlowDown" || /reduce your request rate/i.test(message)) return "s3_throttled";
+	if (permanent) return "permanent";
+	if (/returned 404/.test(message)) return "not_found";
+	if (name === "TimeoutError" || /timed? ?out/i.test(message)) return "timeout";
+	return "other";
+}
 function workerMetrics(metrics) {
 	return {
 		jobDuration: metrics.histogram("registry_job_duration_seconds", "Time to run a job, by kind and outcome (done, retry, failed)", JOB_BUCKETS),
+		jobErrors: metrics.counter("registry_job_errors_total", "Job failures, by kind and reason (s3_throttled, not_found, timeout, permanent, other)"),
 		reconcileQueued: metrics.counter("registry_reconcile_queued_total", "Jobs reconcile queued, by pass (quick, full)"),
 		reconcileRuns: metrics.counter("registry_reconcile_runs_total", "Reconcile passes, by pass (quick, full)")
 	};
@@ -4859,6 +4981,11 @@ const FULL_MISSING = `
     LEFT JOIN registry_versions r ON r.package = m.name AND r.version = v.version
    WHERE m.updated_at < now() - interval '30 seconds' AND ${PUBLISHED}
      AND (r.package IS NULL
+       OR (r.status = 'failed' AND NOT r.permanent
+           AND r.updated_at < now() - interval '1 hour'
+           AND NOT EXISTS (SELECT 1 FROM registry_jobs j
+                            WHERE j.kind = 'process' AND j.package = m.name
+                              AND j.version = v.version))
        OR (r.status = 'pending' AND r.updated_at < now() - interval '5 minutes'
            AND NOT EXISTS (SELECT 1 FROM registry_jobs j
                             WHERE j.kind = 'process' AND j.package = m.name
@@ -4907,7 +5034,10 @@ function publisherOf(job) {
 async function runJob(ctx, job, options) {
 	const finish = (tx) => complete(tx, job);
 	if (job.priority === 10 && await publishedHere(ctx.db, job.package) === false) {
-		await ctx.db.transaction(finish);
+		await ctx.db.transaction(async (tx) => {
+			await tx.query("UPDATE registry_packages SET local = false WHERE name = $1 AND local", [job.package]);
+			await finish(tx);
+		});
 		return;
 	}
 	if (job.kind === "process") {
@@ -4978,10 +5108,14 @@ async function startWorker(ctx, options) {
 				console.error(`[registry] ${job.kind} ${job.package}${job.version ? `@${job.version}` : ""} failed (attempt ${job.attempts}):`, message);
 				const finalAttempt = err instanceof PermanentError || job.attempts >= 5;
 				observe(finalAttempt ? "failed" : "retry");
+				options.metrics?.jobErrors.inc({
+					kind: job.kind,
+					reason: errorReason(err, err instanceof PermanentError)
+				});
 				try {
 					if (!finalAttempt) await retry(ctx.db, job, message);
 					else {
-						if (job.kind === "process") await failVersion(ctx, job, message);
+						if (job.kind === "process") await failVersion(ctx, job, message, err instanceof PermanentError);
 						await complete(ctx.db, job);
 					}
 				} catch (recordErr) {
@@ -5051,7 +5185,8 @@ function s3From(args) {
 		accessKeyId: args.s3AccessKeyId,
 		secretAccessKey: args.s3SecretAccessKey,
 		keyPrefix: args.s3KeyPrefix,
-		s3ForcePathStyle: args.s3ForcePathStyle
+		s3ForcePathStyle: args.s3ForcePathStyle,
+		maxWritesPerSecond: args.s3MaxWritesPerSecond
 	};
 }
 function webhooksFrom(list) {
@@ -5244,12 +5379,15 @@ async function runRegistry(args) {
 		console.log(`  Data:     ${cdnCachePath}`);
 		if (config.s3) console.log(`  S3:       ${config.s3.endpoint}/${config.s3.bucket}`);
 		if (config.renown) console.log(`  Renown auth: ${config.renown.publicUrl}`);
-		if (workers > 0) worker = startWorker(processorContext(runtime, localUrl), {
-			concurrency: workers,
-			webhooks: runtime.webhooks,
-			sweep: true,
-			metrics: jobMetrics
-		});
+		if (workers > 0) {
+			catalogMetrics(metrics, runtime.db);
+			worker = startWorker(processorContext(runtime, localUrl), {
+				concurrency: workers,
+				webhooks: runtime.webhooks,
+				sweep: true,
+				metrics: jobMetrics
+			});
+		}
 	});
 	server.keepAliveTimeout = 65e3;
 	server.headersTimeout = 66e3;
@@ -5290,6 +5428,7 @@ async function runWorker(args) {
 	if (args.metricsPort !== void 0) {
 		processMetrics(metrics);
 		databaseMetrics(metrics, runtime.db);
+		catalogMetrics(metrics, runtime.db);
 		metricsServer = await serveMetrics(metrics, args.metricsPort);
 	}
 	const worker = await startWorker(processorContext(runtime, () => args.registryUrl.replace(/\/$/, "")), {
@@ -5333,8 +5472,10 @@ async function readStoredManifest(s3, config, name) {
 			Key: `${config.keyPrefix ?? ""}${name}/package.json`
 		}));
 		return JSON.parse(await res.Body?.transformToString() ?? "null");
-	} catch {
-		return null;
+	} catch (err) {
+		const name = err.name;
+		if (name === "NoSuchKey" || name === "NotFound") return null;
+		throw err;
 	}
 }
 /** Moves Verdaccio's S3-file state into the Postgres store the S3 plugin reads. */
@@ -5364,15 +5505,21 @@ async function runImportVerdaccioState(args) {
 	const names = new Set(legacy.list ?? []);
 	const pool = new pg.Pool({ connectionString: args.databaseUrl });
 	let pruned = [];
+	let gone = 0;
 	try {
 		const store = new S3DatabasePostgres(pool, pluginLogger);
 		await store.init();
+		const missing = [];
 		for (const name of names) {
 			const manifest = await readStoredManifest(s3, s3Config, name);
 			if (manifest) await store.record(name, manifest);
+			else missing.push(name);
 		}
+		for (const name of missing) names.delete(name);
+		await pool.query("DELETE FROM verdaccio_packages WHERE name = ANY($1)", [missing]);
+		gone = missing.length;
 		for (const name of names) await pool.query("INSERT INTO verdaccio_packages (name) VALUES ($1) ON CONFLICT (name) DO NOTHING", [name]);
-		pruned = await pruneCachedPackages(pool, [...names]);
+		if (legacy.list) pruned = await pruneCachedPackages(pool, [...names]);
 		for (const token of legacy.tokens ?? []) await pool.query(`INSERT INTO verdaccio_tokens ("user", key, token) VALUES ($1, $2, $3)
          ON CONFLICT ("user", key) DO NOTHING`, [
 			token.user,
@@ -5384,10 +5531,14 @@ async function runImportVerdaccioState(args) {
 	} finally {
 		await pool.end();
 	}
-	console.log(`[registry] imported ${names.size} package name(s), ${legacy.tokens?.length ?? 0} token(s)${legacy.secret ? " and the secret" : ""}; pruned ${pruned.length} cached package(s)`);
+	console.log(`[registry] imported ${names.size} package name(s), ${legacy.tokens?.length ?? 0} token(s)${legacy.secret ? " and the secret" : ""}; pruned ${pruned.length} cached package(s), dropped ${gone} with no stored manifest`);
 }
 //#endregion
 //#region cli.ts
+const positiveFromEnv = (name) => () => {
+	const n = Number(process.env[name]);
+	return Number.isInteger(n) && n > 0 ? n : void 0;
+};
 const storageArgs = {
 	storageDir: option({
 		long: "storage-dir",
@@ -5443,6 +5594,12 @@ const storageArgs = {
 		defaultValue: () => process.env.S3_FORCE_PATH_STYLE !== "false",
 		defaultValueIsSerializable: true
 	}),
+	s3MaxWritesPerSecond: option({
+		long: "s3-max-writes-per-second",
+		type: optional(number),
+		description: "Artifact uploads per second (default 150); Hetzner throttles a bucket past 750 requests per second. Env PH_REGISTRY_S3_MAX_WRITES_PER_SECOND",
+		defaultValue: positiveFromEnv("PH_REGISTRY_S3_MAX_WRITES_PER_SECOND")
+	}),
 	databaseUrl: option({
 		long: "database-url",
 		type: optional(string),
@@ -5473,10 +5630,6 @@ const migrateOnBootArg = option({
 	}),
 	description: "Apply database migrations at startup (true/false). Defaults to true without --database-url, else false; env PH_REGISTRY_MIGRATE_ON_BOOT"
 });
-const positiveFromEnv = (name) => () => {
-	const n = Number(process.env[name]);
-	return Number.isInteger(n) && n > 0 ? n : void 0;
-};
 const dbPoolMaxArg = option({
 	long: "db-pool-max",
 	type: optional(number),