construct-hub 0.3.337 → 0.3.340

package/.jsii

@@ -15450,6 +15450,6 @@
       "symbolId": "src/package-sources/npmjs:NpmJsProps"
     }
   },
-  "version": "0.3.337",
-  "fingerprint": "GiBg+z52QoKEcOp9tICbMbk1KURmFxGp37AO5v7djfo="
+  "version": "0.3.340",
+  "fingerprint": "S0Q8aQnP/TyHexqnVHykEwOLUKH5iJiWWTDTJwjrQq4="
 }

package/changelog.md

@@ -1,2 +1,2 @@
 
-### [0.3.337](https://github.com/cdklabs/construct-hub/compare/v0.3.336...v0.3.337) (2022-05-01)
+### [0.3.340](https://github.com/cdklabs/construct-hub/compare/v0.3.339...v0.3.340) (2022-05-02)

package/lib/backend/inventory/canary.lambda.bundle/index.js

@@ -9577,9 +9577,14 @@ var require_lib2 = __commonJS({
 // src/backend/inventory/canary.lambda.ts
 var canary_lambda_exports = {};
 __export(canary_lambda_exports, {
-  handler: () => handler
+  DocumentationStatus: () => DocumentationStatus,
+  Grain: () => Grain,
+  deserialize: () => deserialize,
+  handler: () => handler,
+  serialize: () => serialize
 });
 module.exports = __toCommonJS(canary_lambda_exports);
+var import_zlib2 = require("zlib");
 var import_aws_embedded_metrics = __toESM(require_lib());
 var import_semver = __toESM(require_semver2());
 
@@ -9707,83 +9712,140 @@ var LANGUAGE_DIMENSION = "Language";
 import_aws_embedded_metrics.Configuration.namespace = METRICS_NAMESPACE;
 async function handler(event, context) {
   console.log("Event:", JSON.stringify(event, null, 2));
-  const indexedPackages = /* @__PURE__ */ new Map();
-  const packageNames = /* @__PURE__ */ new Set();
-  const packageMajorVersions = /* @__PURE__ */ new Set();
-  const perLanguage = /* @__PURE__ */ new Map();
+  const scratchworkBucket = requireEnv("SCRATCHWORK_BUCKET_NAME");
+  const { continuationToken, indexedPackages, packageNames, packageMajorVersions, perLanguage } = event.continuationObjectKey ? await loadProgress(event.continuationObjectKey) : {
+    continuationToken: void 0,
+    indexedPackages: /* @__PURE__ */ new Map(),
+    packageNames: /* @__PURE__ */ new Set(),
+    packageMajorVersions: /* @__PURE__ */ new Set(),
+    perLanguage: /* @__PURE__ */ new Map()
+  };
+  async function loadProgress(continuationObjectKey) {
+    console.log("Found a continuation object key, retrieving data from the existing run...");
+    let { Body, ContentEncoding } = await s3().getObject({
+      Bucket: scratchworkBucket,
+      Key: continuationObjectKey
+    }).promise();
+    if (ContentEncoding === "gzip") {
+      Body = (0, import_zlib2.gunzipSync)(Buffer.from(Body));
+    }
+    if (!Body) {
+      throw new Error(`Object key "${event.continuationObjectKey}" not found in bucket "${scratchworkBucket}".`);
+    }
+    console.log("Deserializing data...");
+    const serializedState = Body.toString("utf-8");
+    const state = deserialize(serializedState);
+    console.log("Deserializing finished.");
+    return state;
+  }
   function recordPerLanguage(language, status, pkgName, pkgMajor, pkgVersion, submodule) {
     for (const lang of DocumentationLanguage.ALL) {
-      doRecordPerLanguage(perLanguage, lang, lang === language ? status : PerLanguageStatus.MISSING, pkgName, pkgMajor, pkgVersion, submodule);
-    }
-  }
-  const bucket = requireEnv("BUCKET_NAME");
-  for await (const key of relevantObjectKeys(bucket)) {
-    const [, name, version] = STORAGE_KEY_FORMAT_REGEX.exec(key);
-    packageNames.add(name);
-    const majorVersion = `${name}@${new import_semver.SemVer(version).major}`;
-    packageMajorVersions.add(majorVersion);
-    const fullName = `${name}@${version}`;
-    for (const language of DocumentationLanguage.ALL) {
-      recordPerLanguage(language, PerLanguageStatus.MISSING, name, majorVersion, fullName);
-    }
-    if (!indexedPackages.has(fullName)) {
-      indexedPackages.set(fullName, {});
-    }
-    const status = indexedPackages.get(fullName);
-    if (key.endsWith(METADATA_KEY_SUFFIX)) {
-      status.metadataPresent = true;
-    } else if (key.endsWith(PACKAGE_KEY_SUFFIX)) {
-      status.tarballPresent = true;
-    } else if (key.endsWith(ASSEMBLY_KEY_SUFFIX)) {
-      status.assemblyPresent = true;
-    } else if (key.endsWith(UNINSTALLABLE_PACKAGE_SUFFIX)) {
-      status.uninstallable = true;
-    } else {
-      let identified = false;
+      doRecordPerLanguage(perLanguage, lang, lang === language ? status : DocumentationStatus.MISSING, pkgName, pkgMajor, pkgVersion, submodule);
+    }
+  }
+  async function saveProgress(latestContinuationToken) {
+    console.log("Serializing data...");
+    const serializedState = serialize({
+      continuationToken: latestContinuationToken,
+      packageNames,
+      packageMajorVersions,
+      indexedPackages,
+      perLanguage
+    });
+    console.log("Serializing finished.");
+    const { buffer, contentEncoding } = compressContent(Buffer.from(serializedState));
+    const keyName = `inventory-canary-progress-${Date.now()}`;
+    await s3().putObject({
+      Bucket: scratchworkBucket,
+      Key: keyName,
+      Body: buffer,
+      ContentType: "application/json",
+      ContentEncoding: contentEncoding,
+      Metadata: {
+        "Lambda-Log-Group": context.logGroupName,
+        "Lambda-Log-Stream": context.logStreamName,
+        "Lambda-Run-Id": context.awsRequestId
+      }
+    }).promise();
+    return {
+      continuationObjectKey: keyName
+    };
+  }
+  const maxMetricProcessingTime = 6e4;
+  const packageDataBucket = requireEnv("PACKAGE_DATA_BUCKET_NAME");
+  for await (const [keys, latestContinuationToken] of relevantObjectKeys(packageDataBucket, continuationToken)) {
+    for (const key of keys) {
+      const [, name, version] = STORAGE_KEY_FORMAT_REGEX.exec(key);
+      packageNames.add(name);
+      const majorVersion = `${name}@${new import_semver.SemVer(version).major}`;
+      packageMajorVersions.add(majorVersion);
+      const fullName = `${name}@${version}`;
       for (const language of DocumentationLanguage.ALL) {
-        const matchJson = submoduleKeyRegexp(language, "json").exec(key);
-        const matchMd = submoduleKeyRegexp(language, "md").exec(key);
-        if (matchJson != null) {
-          const [, submodule, isUnsupported] = matchJson;
-          if (status.submodules == null) {
-            status.submodules = /* @__PURE__ */ new Set();
-          }
-          status.submodules.add(`${fullName}.${submodule}`);
-          recordPerLanguage(language, isUnsupported ? PerLanguageStatus.UNSUPPORTED : PerLanguageStatus.SUPPORTED, name, majorVersion, fullName, submodule);
-          identified = true;
-        } else if (key.endsWith(docsKeySuffix(language, void 0, "json"))) {
-          recordPerLanguage(language, PerLanguageStatus.SUPPORTED, name, majorVersion, fullName);
-          identified = true;
-        } else if (key.endsWith(notSupportedKeySuffix(language, void 0, "json"))) {
-          recordPerLanguage(language, PerLanguageStatus.UNSUPPORTED, name, majorVersion, fullName);
-          identified = true;
-        } else if (key.endsWith(corruptAssemblyKeySuffix(language, void 0, "json"))) {
-          recordPerLanguage(language, PerLanguageStatus.CORRUPT_ASSEMBLY, name, majorVersion, fullName);
-          identified = true;
-        } else if (matchMd != null) {
-          identified = true;
-        } else if (key.endsWith(docsKeySuffix(language, void 0, "md"))) {
-          identified = true;
-        } else if (key.endsWith(notSupportedKeySuffix(language, void 0, "md"))) {
-          identified = true;
-        } else if (key.endsWith(corruptAssemblyKeySuffix(language, void 0, "md"))) {
-          identified = true;
-        }
-      }
-      if (!identified) {
-        status.unknownObjects = status.unknownObjects ?? [];
-        status.unknownObjects.push(key);
-      }
+        recordPerLanguage(language, DocumentationStatus.MISSING, name, majorVersion, fullName);
+      }
+      if (!indexedPackages.has(fullName)) {
+        indexedPackages.set(fullName, {});
+      }
+      const status = indexedPackages.get(fullName);
+      if (key.endsWith(METADATA_KEY_SUFFIX)) {
+        status.metadataPresent = true;
+      } else if (key.endsWith(PACKAGE_KEY_SUFFIX)) {
+        status.tarballPresent = true;
+      } else if (key.endsWith(ASSEMBLY_KEY_SUFFIX)) {
+        status.assemblyPresent = true;
+      } else if (key.endsWith(UNINSTALLABLE_PACKAGE_SUFFIX)) {
+        status.uninstallable = true;
+      } else {
+        let identified = false;
+        for (const language of DocumentationLanguage.ALL) {
+          const matchJson = submoduleKeyRegexp(language, "json").exec(key);
+          const matchMd = submoduleKeyRegexp(language, "md").exec(key);
+          if (matchJson != null) {
+            const [, submodule, isUnsupported] = matchJson;
+            if (status.submodules == null) {
+              status.submodules = /* @__PURE__ */ new Set();
+            }
+            status.submodules.add(`${fullName}.${submodule}`);
+            recordPerLanguage(language, isUnsupported ? DocumentationStatus.UNSUPPORTED : DocumentationStatus.SUPPORTED, name, majorVersion, fullName, submodule);
+            identified = true;
+          } else if (key.endsWith(docsKeySuffix(language, void 0, "json"))) {
+            recordPerLanguage(language, DocumentationStatus.SUPPORTED, name, majorVersion, fullName);
+            identified = true;
+          } else if (key.endsWith(notSupportedKeySuffix(language, void 0, "json"))) {
+            recordPerLanguage(language, DocumentationStatus.UNSUPPORTED, name, majorVersion, fullName);
+            identified = true;
+          } else if (key.endsWith(corruptAssemblyKeySuffix(language, void 0, "json"))) {
+            recordPerLanguage(language, DocumentationStatus.CORRUPT_ASSEMBLY, name, majorVersion, fullName);
+            identified = true;
+          } else if (matchMd != null) {
+            identified = true;
+          } else if (key.endsWith(docsKeySuffix(language, void 0, "md"))) {
+            identified = true;
+          } else if (key.endsWith(notSupportedKeySuffix(language, void 0, "md"))) {
+            identified = true;
+          } else if (key.endsWith(corruptAssemblyKeySuffix(language, void 0, "md"))) {
+            identified = true;
+          }
+        }
+        if (!identified) {
+          status.unknownObjects = status.unknownObjects ?? [];
+          status.unknownObjects.push(key);
+        }
+      }
+    }
+    if (latestContinuationToken && context.getRemainingTimeInMillis() <= maxMetricProcessingTime) {
+      console.log("Running up to the Lambda time limit and there are still items to process. Saving our current progress...");
+      return saveProgress(latestContinuationToken);
     }
   }
   const reports = [];
   function createReport(reportKey, packageVersions) {
     const report = JSON.stringify(packageVersions, null, 2);
     const { buffer, contentEncoding } = compressContent(Buffer.from(report));
-    console.log(`Uploading list to s3://${bucket}/${reportKey}`);
+    console.log(`Uploading list to s3://${packageDataBucket}/${reportKey}`);
     reports.push(s3().putObject({
       Body: buffer,
-      Bucket: bucket,
+      Bucket: packageDataBucket,
       ContentEncoding: contentEncoding,
       ContentType: "application/json",
       Expires: new Date(Date.now() + 3e5),
@@ -9843,16 +9905,17 @@ async function handler(event, context) {
       console.log(`### Start of data for ${language}`);
       metrics.setDimensions({ [LANGUAGE_DIMENSION]: language.toString() });
       for (const forStatus of [
-        PerLanguageStatus.SUPPORTED,
-        PerLanguageStatus.UNSUPPORTED,
-        PerLanguageStatus.MISSING,
-        PerLanguageStatus.CORRUPT_ASSEMBLY
+        DocumentationStatus.SUPPORTED,
+        DocumentationStatus.UNSUPPORTED,
+        DocumentationStatus.MISSING,
+        DocumentationStatus.CORRUPT_ASSEMBLY
       ]) {
-        for (const [key, statuses] of Object.entries(data)) {
+        for (const [key, statuses] of data.entries()) {
           let filtered = Array.from(statuses.entries()).filter(([, status]) => forStatus === status);
           let metricName = METRIC_NAME_BY_STATUS_AND_GRAIN[forStatus][key];
-          if (forStatus === PerLanguageStatus.MISSING && metricName === "MissingPackageVersionCount" /* PER_LANGUAGE_MISSING_VERSIONS */ || forStatus === PerLanguageStatus.CORRUPT_ASSEMBLY && metricName === "CorruptAssemblyPackageVersionCount" /* PER_LANGUAGE_CORRUPT_ASSEMBLY_VERSIONS */) {
-            const reportKey = forStatus === PerLanguageStatus.MISSING ? missingDocumentationReport(language) : corruptAssemblyReport(language);
+          if (forStatus === DocumentationStatus.MISSING && metricName === "MissingPackageVersionCount" /* PER_LANGUAGE_MISSING_VERSIONS */ || forStatus === DocumentationStatus.CORRUPT_ASSEMBLY && metricName === "CorruptAssemblyPackageVersionCount" /* PER_LANGUAGE_CORRUPT_ASSEMBLY_VERSIONS */) {
+            const lang = DocumentationLanguage.fromString(language);
+            const reportKey = forStatus === DocumentationStatus.MISSING ? missingDocumentationReport(lang) : corruptAssemblyReport(lang);
             createReport(reportKey, filtered.map(([name]) => name).sort());
           }
           console.log(`${forStatus} ${key} for ${language}: ${filtered.length} entries`);
@@ -9867,20 +9930,25 @@ async function handler(event, context) {
   for (const report of reports) {
     await report;
   }
+  return {};
 }
-async function* relevantObjectKeys(bucket) {
+async function* relevantObjectKeys(bucket, continuationToken) {
   const request = {
     Bucket: bucket,
     Prefix: STORAGE_KEY_PREFIX
   };
+  if (continuationToken)
+    request.ContinuationToken = continuationToken;
   do {
     const response = await s3().listObjectsV2(request).promise();
+    const keys = [];
     for (const { Key } of response.Contents ?? []) {
       if (Key == null) {
         continue;
       }
-      yield Key;
+      keys.push(Key);
     }
+    yield [keys, response.ContinuationToken];
     request.ContinuationToken = response.NextContinuationToken;
   } while (request.ContinuationToken != null);
 }
@@ -9892,13 +9960,20 @@ function submoduleKeyRegexp(language, fileExt) {
     return str.replace(/([+*.()?$[\]])/g, "\\$1");
   }
 }
-var PerLanguageStatus = /* @__PURE__ */ ((PerLanguageStatus2) => {
-  PerLanguageStatus2["MISSING"] = "Missing";
-  PerLanguageStatus2["UNSUPPORTED"] = "Unsupported";
-  PerLanguageStatus2["CORRUPT_ASSEMBLY"] = "CorruptAssembly";
-  PerLanguageStatus2["SUPPORTED"] = "Supported";
-  return PerLanguageStatus2;
-})(PerLanguageStatus || {});
+var Grain = /* @__PURE__ */ ((Grain2) => {
+  Grain2["PACKAGE_MAJOR_VERSIONS"] = "package major versions";
+  Grain2["PACKAGE_VERSION_SUBMODULES"] = "package version submodules";
+  Grain2["PACKAGE_VERSIONS"] = "package versions";
+  Grain2["PACKAGES"] = "packages";
+  return Grain2;
+})(Grain || {});
+var DocumentationStatus = /* @__PURE__ */ ((DocumentationStatus2) => {
+  DocumentationStatus2["MISSING"] = "Missing";
+  DocumentationStatus2["UNSUPPORTED"] = "Unsupported";
+  DocumentationStatus2["CORRUPT_ASSEMBLY"] = "CorruptAssembly";
+  DocumentationStatus2["SUPPORTED"] = "Supported";
+  return DocumentationStatus2;
+})(DocumentationStatus || {});
 var METRIC_NAME_BY_STATUS_AND_GRAIN = {
   ["Missing" /* MISSING */]: {
     ["packages" /* PACKAGES */]: "MissingPackageCount" /* PER_LANGUAGE_MISSING_PACKAGES */,
@@ -9926,21 +10001,21 @@ var METRIC_NAME_BY_STATUS_AND_GRAIN = {
   }
 };
 function doRecordPerLanguage(perLanguage, language, status, pkgName, pkgMajor, pkgVersion, submodule) {
-  if (!perLanguage.has(language)) {
-    perLanguage.set(language, {
-      ["package major versions" /* PACKAGE_MAJOR_VERSIONS */]: /* @__PURE__ */ new Map(),
-      ["packages" /* PACKAGES */]: /* @__PURE__ */ new Map(),
-      ["package version submodules" /* PACKAGE_VERSION_SUBMODULES */]: /* @__PURE__ */ new Map(),
-      ["package versions" /* PACKAGE_VERSIONS */]: /* @__PURE__ */ new Map()
-    });
-  }
-  const data = perLanguage.get(language);
+  if (!perLanguage.has(language.name)) {
+    const perGrainData = /* @__PURE__ */ new Map();
+    perGrainData.set("package major versions" /* PACKAGE_MAJOR_VERSIONS */, /* @__PURE__ */ new Map());
+    perGrainData.set("packages" /* PACKAGES */, /* @__PURE__ */ new Map());
+    perGrainData.set("package version submodules" /* PACKAGE_VERSION_SUBMODULES */, /* @__PURE__ */ new Map());
+    perGrainData.set("package versions" /* PACKAGE_VERSIONS */, /* @__PURE__ */ new Map());
+    perLanguage.set(language.name, perGrainData);
+  }
+  const data = perLanguage.get(language.name);
   const outputDomains = submodule ? [
-    [data["package version submodules" /* PACKAGE_VERSION_SUBMODULES */], `${pkgVersion}.${submodule}`]
+    [data.get("package version submodules" /* PACKAGE_VERSION_SUBMODULES */), `${pkgVersion}.${submodule}`]
   ] : [
-    [data["package major versions" /* PACKAGE_MAJOR_VERSIONS */], pkgMajor],
-    [data["package versions" /* PACKAGE_VERSIONS */], pkgVersion],
-    [data["packages" /* PACKAGES */], pkgName]
+    [data.get("package major versions" /* PACKAGE_MAJOR_VERSIONS */), pkgMajor],
+    [data.get("package versions" /* PACKAGE_VERSIONS */), pkgVersion],
+    [data.get("packages" /* PACKAGES */), pkgName]
   ];
   for (const [map, name] of outputDomains) {
     switch (status) {
@@ -9961,8 +10036,44 @@ function doRecordPerLanguage(perLanguage, language, status, pkgName, pkgMajor, p
     }
   }
 }
+function safeReplacer(_key, value) {
+  if (value instanceof Map) {
+    return {
+      _type: "Map",
+      value: Array.from(value.entries())
+    };
+  } else if (value instanceof Set) {
+    return {
+      _type: "Set",
+      value: Array.from([...value])
+    };
+  } else {
+    return value;
+  }
+}
+function safeReviver(_key, value) {
+  if (typeof value === "object" && value !== null) {
+    if (value._type === "Map") {
+      return new Map(value.value);
+    }
+    if (value._type === "Set") {
+      return new Set(value.value);
+    }
+  }
+  return value;
+}
+function serialize(value) {
+  return JSON.stringify(value, safeReplacer, 2);
+}
+function deserialize(value) {
+  return JSON.parse(value, safeReviver);
+}
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
-  handler
+  DocumentationStatus,
+  Grain,
+  deserialize,
+  handler,
+  serialize
 });
 //# sourceMappingURL=index.js.map