@powerhousedao/registry 6.2.3-dev.36 → 6.2.3-dev.38

package/dist/cli.d.mts

@@ -22,6 +22,7 @@ export declare const registryCommand: Partial<import("cmd-ts/dist/cjs/argparser.
     workers: number;
     uplink: string | undefined;
     uplinkMaxage: string | undefined;
+    trustProxy: string | undefined;
     webEnabled: boolean;
     publicUrl: string | undefined;
     authRenown: boolean;
@@ -51,6 +52,7 @@ export declare const registryCommand: Partial<import("cmd-ts/dist/cjs/argparser.
   workers: number;
   uplink: string | undefined;
   uplinkMaxage: string | undefined;
+  trustProxy: string | undefined;
   webEnabled: boolean;
   publicUrl: string | undefined;
   authRenown: boolean;

package/dist/cli.d.mts.map

@@ -1 +1 @@
-{"version":3,"file":"cli.d.mts","names":[],"sources":["../cli.ts"],"mappings":";qBA2Ia,iBAAe,+CAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAkI1B,6CAAA,iDAAA,mDAAA;qBAEW,eAAa,+CAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAyCxB,6CAAA,iDAAA,mDAAA;qBAEW,gBAAc,+CAAA;;;;;;;;;;;;IAkBzB,6CAAA,iDAAA,mDAAA;qBAEW,6BAA2B,+CAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAatC,6CAAA,iDAAA,mDAAA"}
+{"version":3,"file":"cli.d.mts","names":[],"sources":["../cli.ts"],"mappings":";qBA2Ia,iBAAe,+CAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IA2I1B,6CAAA,iDAAA,mDAAA;qBAEW,eAAa,+CAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAyCxB,6CAAA,iDAAA,mDAAA;qBAEW,gBAAc,+CAAA;;;;;;;;;;;;IAkBzB,6CAAA,iDAAA,mDAAA;qBAEW,6BAA2B,+CAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAatC,6CAAA,iDAAA,mDAAA"}

package/dist/cli.mjs

@@ -1919,13 +1919,19 @@ function createS3ArtifactStore(config) {
 		},
 		async putFile(key, file, contentType) {
 			const { size } = await fs.promises.stat(file);
-			await s3.send(new PutObjectCommand({
-				Bucket: config.bucket,
-				Key: root + key,
-				Body: fs.createReadStream(file),
-				ContentLength: size,
-				ContentType: contentType
-			}));
+			const body = fs.createReadStream(file);
+			body.on("error", () => void 0);
+			try {
+				await s3.send(new PutObjectCommand({
+					Bucket: config.bucket,
+					Key: root + key,
+					Body: body,
+					ContentLength: size,
+					ContentType: contentType
+				}));
+			} finally {
+				body.destroy();
+			}
 		},
 		async get(key) {
 			try {
@@ -2826,9 +2832,16 @@ function inside(root, relative) {
 }
 async function inBatches(items, size, fn) {
 	let cursor = 0;
-	await Promise.all(Array.from({ length: Math.min(size, items.length) }, async () => {
-		while (cursor < items.length) await fn(items[cursor++]);
-	}));
+	let failed = false;
+	const rejected = (await Promise.allSettled(Array.from({ length: Math.min(size, items.length) }, async () => {
+		while (!failed && cursor < items.length) try {
+			await fn(items[cursor++]);
+		} catch (err) {
+			failed = true;
+			throw err;
+		}
+	}))).find((lane) => lane.status === "rejected");
+	if (rejected) throw rejected.reason;
 }
 async function packDir(dir) {
 	const files = listFiles(dir);
@@ -2933,6 +2946,12 @@ async function indexedRevision(db, pkg) {
 	}
 	return (await db.query("SELECT rev FROM verdaccio_manifests WHERE name = $1", [pkg])).rows[0]?.rev ?? null;
 }
+async function publishedHere(db, pkg) {
+	if (!(await db.query("SELECT to_regclass('verdaccio_packages') IS NOT NULL AS exists")).rows[0]?.exists) return null;
+	return (await db.query(`SELECT EXISTS (SELECT 1 FROM verdaccio_packages WHERE name = $1)
+         OR EXISTS (SELECT 1 FROM registry_package_owners WHERE package_name = $1)
+         AS published`, [pkg])).rows[0]?.published === true;
+}
 async function upsertPackage(tx, pkg, packument, local, manifestRev) {
 	const versions = Object.keys(packument.versions ?? {}).sort(compareSemver);
 	const times = {};
@@ -3017,7 +3036,7 @@ function publisherOf$1(job) {
 async function processVersion(ctx, job, finish) {
 	const pkg = job.package;
 	const version = job.version;
-	const local = job.payload.local === true;
+	const local = await publishedHere(ctx.db, pkg) ?? job.payload.local === true;
 	const manifestRev = await indexedRevision(ctx.db, pkg);
 	const packument = await fetchPackument(ctx.registryUrl, pkg);
 	if (!packument?.versions?.[version]) {
@@ -3102,15 +3121,22 @@ async function syncPackage(ctx, job, finish) {
 	const known = await ctx.db.query("SELECT version FROM registry_versions WHERE package = $1", [pkg]);
 	const listed = new Set(Object.keys(packument?.versions ?? {}));
 	const removed = known.rows.map((r) => r.version).filter((v) => !listed.has(v));
+	const published = await publishedHere(ctx.db, pkg);
+	const local = published ?? job.payload.local === true;
+	if (!packument && published) throw new Error(`metadata for published ${pkg} returned 404`);
 	await ctx.db.transaction(async (tx) => {
 		await removeVersionRows(tx, pkg, removed);
 		if (!packument) await tx.query("DELETE FROM registry_packages WHERE name = $1", [pkg]);
 		else {
-			await upsertPackage(tx, pkg, packument, job.payload.local === true, manifestRev);
+			await upsertPackage(tx, pkg, packument, local, manifestRev);
 			await recomputeLatest(tx, pkg);
-			if ((await tx.query("SELECT local FROM registry_packages WHERE name = $1", [pkg])).rows[0]?.local) {
+			if (local) {
 				const have = new Set(known.rows.map((r) => r.version));
-				for (const version of listed) if (!have.has(version)) await enqueue(tx, "process", pkg, version, { local: true }, job.priority);
+				const tagged = new Set(Object.values(packument["dist-tags"] ?? {}));
+				for (const version of tagged) {
+					if (!listed.has(version)) continue;
+					if (!have.has(version)) await enqueue(tx, "process", pkg, version, { local: true }, job.priority);
+				}
 			}
 		}
 		await finish(tx);
@@ -4761,7 +4787,10 @@ function buildVerdaccioConfig(config) {
 			primary_color: "#38C780",
 			darkMode: true
 		},
-		server: { keepAliveTimeout: 60 },
+		server: {
+			keepAliveTimeout: 60,
+			...config.trustProxy !== void 0 && { trustProxy: config.trustProxy }
+		},
 		log: {
 			type: "stdout",
 			format: "pretty",
@@ -4799,8 +4828,10 @@ async function enqueueSweep(db) {
 	return names.rows.length;
 }
 async function hasManifestIndex(db) {
-	return (await db.query("SELECT to_regclass('verdaccio_manifests') IS NOT NULL AS exists")).rows[0]?.exists === true;
+	return (await db.query(`SELECT to_regclass('verdaccio_manifests') IS NOT NULL
+        AND to_regclass('verdaccio_packages') IS NOT NULL AS exists`)).rows[0]?.exists === true;
 }
+const PUBLISHED = "EXISTS (SELECT 1 FROM verdaccio_packages l WHERE l.name = m.name)";
 const UNPUBLISHED = `
   SELECT p.name FROM registry_packages p
    WHERE p.local AND p.updated_at < now() - interval '30 seconds'
@@ -4809,7 +4840,7 @@ const QUICK_CHANGED = `
   SELECT m.name
     FROM verdaccio_manifests m
     LEFT JOIN registry_packages p ON p.name = m.name
-   WHERE m.updated_at < now() - interval '30 seconds'
+   WHERE m.updated_at < now() - interval '30 seconds' AND ${PUBLISHED}
      AND (p.name IS NULL OR p.manifest_rev IS DISTINCT FROM m.rev)
   UNION ${UNPUBLISHED}
    LIMIT 1000`;
@@ -4822,11 +4853,11 @@ const QUICK_MISSING = `
                         AND j.version = r.version)
    LIMIT 1000`;
 const FULL_MISSING = `
-  SELECT m.name, v.version
+  SELECT DISTINCT m.name, v.version
     FROM verdaccio_manifests m
-    CROSS JOIN LATERAL jsonb_array_elements_text(m.versions) AS v(version)
+    CROSS JOIN LATERAL jsonb_each_text(m.dist_tags) AS v(tag, version)
     LEFT JOIN registry_versions r ON r.package = m.name AND r.version = v.version
-   WHERE m.updated_at < now() - interval '30 seconds'
+   WHERE m.updated_at < now() - interval '30 seconds' AND ${PUBLISHED}
      AND (r.package IS NULL
        OR (r.status = 'pending' AND r.updated_at < now() - interval '5 minutes'
            AND NOT EXISTS (SELECT 1 FROM registry_jobs j
@@ -4837,8 +4868,8 @@ const FULL_CHANGED = `
   SELECT m.name
     FROM verdaccio_manifests m
     LEFT JOIN registry_packages p ON p.name = m.name
-   WHERE m.updated_at < now() - interval '30 seconds'
-     AND (p.name IS NULL OR p.dist_tags <> m.dist_tags
+   WHERE m.updated_at < now() - interval '30 seconds' AND ${PUBLISHED}
+     AND (p.name IS NULL OR NOT p.local OR p.dist_tags <> m.dist_tags
        OR EXISTS (SELECT 1 FROM registry_versions r
                    WHERE r.package = m.name AND NOT m.versions ? r.version))
   UNION ${UNPUBLISHED}
@@ -4853,12 +4884,32 @@ async function reconcile(db, options = {}) {
 	if (queued > 0) await wakeWorkers(db);
 	return queued;
 }
+async function pruneCachedPackages(q, published) {
+	if (!(await q.query("SELECT to_regclass('registry_jobs') IS NOT NULL AS exists")).rows[0]?.exists) return [];
+	const pruned = (await q.query(`DELETE FROM verdaccio_packages p
+      WHERE p.name <> ALL($1::text[])
+        AND NOT EXISTS (SELECT 1 FROM registry_package_owners o
+                         WHERE o.package_name = p.name)
+        AND NOT EXISTS (SELECT 1 FROM registry_packages r
+                         WHERE r.name = p.name AND r.listed_manifest IS NOT NULL)
+      RETURNING p.name`, [published])).rows.map((row) => row.name);
+	if (pruned.length) {
+		await q.query("DELETE FROM registry_jobs WHERE package = ANY($1) AND locked_by IS NULL", [pruned]);
+		await q.query("DELETE FROM registry_versions WHERE package = ANY($1)", [pruned]);
+		await q.query("DELETE FROM registry_packages WHERE name = ANY($1)", [pruned]);
+	}
+	return pruned;
+}
 function publisherOf(job) {
 	const by = job.payload.publishedBy;
 	return by?.address ? by : void 0;
 }
 async function runJob(ctx, job, options) {
 	const finish = (tx) => complete(tx, job);
+	if (job.priority === 10 && await publishedHere(ctx.db, job.package) === false) {
+		await ctx.db.transaction(finish);
+		return;
+	}
 	if (job.kind === "process") {
 		if (await processVersion(ctx, job, finish) && job.payload.notify === true) await options.webhooks?.notifyPublish({
 			packageName: job.package,
@@ -4942,7 +4993,7 @@ async function startWorker(ctx, options) {
 	};
 	const loops = Array.from({ length: Math.max(1, options.concurrency) }, () => loop());
 	let sweeper;
-	let lastFull = 0;
+	let lastFull = Date.now() - SWEEP_INTERVAL_MS + RECONCILE_INTERVAL_MS;
 	const sweep = async () => {
 		try {
 			const full = Date.now() - lastFull >= SWEEP_INTERVAL_MS;
@@ -5057,8 +5108,11 @@ async function seedVerdaccioSecret(databaseUrl, secret) {
 const MIB = 1048576;
 const DRAIN_DELAY_MS = 5e3;
 const DRAIN_TIMEOUT_MS = 2e4;
+function parseTrustProxy(value) {
+	return /^\d+$/.test(value) ? Number(value) : value;
+}
 async function runRegistry(args) {
-	const { port, storageDir, cdnCacheDir, uplink, uplinkMaxage, webEnabled, webhooks, publicUrl, authRenown, renownUrl, verdaccioSecret, localPackages, databaseUrl, pluginsDir, authStore } = args;
+	const { port, storageDir, cdnCacheDir, uplink, uplinkMaxage, trustProxy, webEnabled, webhooks, publicUrl, authRenown, renownUrl, verdaccioSecret, localPackages, databaseUrl, pluginsDir, authStore } = args;
 	const storagePath = await resolveDir(storageDir);
 	const cdnCachePath = await resolveDir(cdnCacheDir);
 	const s3 = s3From(args);
@@ -5084,6 +5138,7 @@ async function runRegistry(args) {
 		uplink,
 		uplinkMaxage,
 		webEnabled,
+		...trustProxy ? { trustProxy: parseTrustProxy(trustProxy) } : {},
 		...localPackagePatterns?.length ? { localPackagePatterns } : {},
 		...renownEnabled && publicUrl ? { renown: {
 			publicUrl,
@@ -5271,27 +5326,6 @@ async function runMigrate(databaseUrl, options) {
 		await db.close();
 	}
 }
-async function storedPackageNames(s3, config) {
-	const prefix = config.keyPrefix ?? "";
-	const names = /* @__PURE__ */ new Set();
-	let token;
-	do {
-		const page = await s3.send(new ListObjectsV2Command({
-			Bucket: config.bucket,
-			Prefix: prefix,
-			ContinuationToken: token
-		}));
-		for (const { Key } of page.Contents ?? []) {
-			const rel = Key?.slice(prefix.length) ?? "";
-			if (!rel.endsWith("/package.json") || rel.startsWith("artifacts/")) continue;
-			const name = rel.slice(0, -13);
-			const depth = name.split("/").length;
-			if (name.startsWith("@") ? depth === 2 : depth === 1) names.add(name);
-		}
-		token = page.IsTruncated ? page.NextContinuationToken : void 0;
-	} while (token);
-	return names;
-}
 async function readStoredManifest(s3, config, name) {
 	try {
 		const res = await s3.send(new GetObjectCommand({
@@ -5327,9 +5361,9 @@ async function runImportVerdaccioState(args) {
 		const name = err.name;
 		if (name !== "NoSuchKey" && name !== "NotFound") throw err;
 	}
-	const names = await storedPackageNames(s3, s3Config);
-	for (const name of legacy.list ?? []) names.add(name);
+	const names = new Set(legacy.list ?? []);
 	const pool = new pg.Pool({ connectionString: args.databaseUrl });
+	let pruned = [];
 	try {
 		const store = new S3DatabasePostgres(pool, pluginLogger);
 		await store.init();
@@ -5338,6 +5372,7 @@ async function runImportVerdaccioState(args) {
 			if (manifest) await store.record(name, manifest);
 		}
 		for (const name of names) await pool.query("INSERT INTO verdaccio_packages (name) VALUES ($1) ON CONFLICT (name) DO NOTHING", [name]);
+		pruned = await pruneCachedPackages(pool, [...names]);
 		for (const token of legacy.tokens ?? []) await pool.query(`INSERT INTO verdaccio_tokens ("user", key, token) VALUES ($1, $2, $3)
          ON CONFLICT ("user", key) DO NOTHING`, [
 			token.user,
@@ -5349,7 +5384,7 @@ async function runImportVerdaccioState(args) {
 	} finally {
 		await pool.end();
 	}
-	console.log(`[registry] imported ${names.size} package name(s), ${legacy.tokens?.length ?? 0} token(s)${legacy.secret ? " and the secret" : ""}`);
+	console.log(`[registry] imported ${names.size} package name(s), ${legacy.tokens?.length ?? 0} token(s)${legacy.secret ? " and the secret" : ""}; pruned ${pruned.length} cached package(s)`);
 }
 //#endregion
 //#region cli.ts
@@ -5500,6 +5535,13 @@ const registryCommand = command({
 			defaultValue: () => process.env.PH_REGISTRY_UPLINK_MAXAGE,
 			defaultValueIsSerializable: true
 		}),
+		trustProxy: option({
+			long: "trust-proxy",
+			type: optional(string),
+			description: "Express `trust proxy` (e.g. 2 proxy hops), so rate limits key on client IPs; env PH_REGISTRY_TRUST_PROXY",
+			defaultValue: () => process.env.PH_REGISTRY_TRUST_PROXY,
+			defaultValueIsSerializable: true
+		}),
 		webEnabled: flag({
 			long: "web-enabled",
 			defaultValue: () => process.env.REGISTRY_WEB !== "false",
@@ -5546,6 +5588,7 @@ const registryCommand = command({
 			databaseUrl: redact(args.databaseUrl),
 			listenDatabaseUrl: redact(args.listenDatabaseUrl),
 			verdaccioSecret: redact(args.verdaccioSecret),
+			s3AccessKeyId: redact(args.s3AccessKeyId),
 			s3SecretAccessKey: redact(args.s3SecretAccessKey)
 		});
 		try {