fyn 1.1.39 → 1.1.40

package/dist/fyn.js

@@ -2690,8 +2690,8 @@ class FynCentral {
     return info.mutates ? false : true;
   }
 
-  async get(integrity) {
-    return await this.getInfo(integrity).contentPath;
+  async getContentPath(integrity) {
+    return (await this.getInfo(integrity)).contentPath;
   }
 
   async getInfo(integrity) {
@@ -2707,21 +2707,10 @@ class FynCentral {
 
     return info;
   }
-  /**
-   * Get the hash of a npm package's extracted content.
-   * - only consider the file names and their mtime and size because npm tar files
-   *   with a fixed timestamp to publish, so the mtime give us some assurance
-   *   to know if file changed.
-   *
-   * @param {*} integrity - shasum integrity for the package
-   * @returns void
-   */
 
-
-  async getContentShasum(integrity) {
+  async _calcContentShasum(info, pkgDir = undefined) {
     try {
-      const info = await this.getInfo(integrity);
-      const packageDir = Path.join(info.contentPath, "package");
+      const packageDir = pkgDir || Path.join(info.contentPath, "package");
 
       const filter = (_file, _path, extras) => {
         const {
@@ -2751,11 +2740,40 @@ class FynCentral {
       return undefined;
     }
   }
+  /**
+   * Get the hash of a npm package's extracted content.
+   * - only consider the file names and their mtime and size because npm tar files
+   *   with a fixed timestamp to publish, so the mtime give us some assurance
+   *   to know if file changed.
+   *
+   * @param {*} integrity - shasum integrity for the package
+   * @returns void
+   */
+
+
+  async getContentShasum(integrity) {
+    try {
+      const info = await this.getInfo(integrity);
+      return this._calcContentShasum(info);
+    } catch (_err) {
+      return undefined;
+    }
+  }
 
   async getMutation(integrity) {
     const info = this._map.has(integrity) ? this._map.get(integrity) : await this._loadTree(integrity);
     return info.mutates;
   }
+
+  async delete(integrity) {
+    const info = this._map.get(integrity);
+
+    if (info && info.exist && info.contentPath) {
+      await Fs.$.rimraf(info.contentPath);
+
+      this._map.delete(integrity);
+    }
+  }
   /**
    * Read the content dir tree from file into into
    *
@@ -2776,6 +2794,7 @@ class FynCentral {
         }
 
         info.tree = tree.$;
+        info.shaSum = tree.shaSum;
       } else {
         info.tree = tree;
       }
@@ -2794,6 +2813,7 @@ class FynCentral {
   async saveInfoTree(info, path) {
     await Fs.writeFile(Path.join(path || info.contentPath, "tree.json"), JSON.stringify({
       $: info.tree,
+      shaSum: info.shaSum,
       mutates: info.mutates,
       _: 1
     }));
@@ -2899,7 +2919,7 @@ class FynCentral {
     return tmpLock;
   }
 
-  async _storeTarStream(info, _stream) {
+  async _storeTarStream(info, integrity, _stream) {
     let stream = _stream;
     const tmp = `${info.contentPath}.tmp`;
     await Fs.$.rimraf(tmp); // in case there was any remnant left from an interrupted install
@@ -2917,10 +2937,40 @@ class FynCentral {
 
 
     info.tree = await this._untarStream(stream, targetDir, info);
+    info.shaSum = await this._calcContentShasum(info, targetDir);
     await this.saveInfoTree(info, tmp);
     await Fs.rename(tmp, info.contentPath);
     info.exist = true;
   }
+  /**
+   * Validate the central store package content by file size and timestamp
+   *
+   * @param {*} integrity - package integrity
+   *
+   * @returns true or false
+   */
+
+
+  async validate(integrity) {
+    const info = this._map.get(integrity);
+
+    if (info === undefined || !info.exist) {
+      return false;
+    }
+
+    if (info.validated === undefined) {
+      const sum = await this.getContentShasum(integrity);
+
+      if (!info.shaSum) {
+        info.shaSum = sum;
+        await this.saveInfoTree(info);
+      }
+
+      info.validated = info.shaSum === sum;
+    }
+
+    return info.validated;
+  }
 
   async storeTarStream(pkgId, integrity, stream) {
     let tmpLock = false;
@@ -2948,7 +2998,7 @@ class FynCentral {
           }
         } else {
           logger.debug("storing tar to central store", pkgId, integrity);
-          await this._storeTarStream(info, stream);
+          await this._storeTarStream(info, integrity, stream);
           stream = undefined; // eslint-disable-line
 
           this._map.set(integrity, info);
@@ -3319,6 +3369,15 @@ class Fyn {
       return semv === false || semv && semv.includes("no-fyn-local");
     });
     return Boolean(noFynLocal);
+  } // are we installing modules for the top level fynpo dir
+
+
+  isTopLevelFynpoInstall() {
+    if (this._fynpo.config) {
+      return this._fynpo.dir === this._cwd;
+    }
+
+    return false;
   }
 
   async _initializePkg() {
@@ -4319,8 +4378,7 @@ module.exports = LifecycleScripts;
  *
  * - TODO: impl timestamp check so if no files updated then don't run
  */
-
-const assert = __webpack_require__("assert");
+// const assert = require("assert");
 
 const Path = __webpack_require__("path");
 
@@ -4455,13 +4513,12 @@ class LocalPkgBuilder {
   }
 
   async waitForItem(fullPath) {
-    if (this._waitItems[fullPath] === undefined) {
+    if (this._waitItems[fullPath] === undefined && this._started.promise) {
       logger.debug("waiting for local build item start, fullPath:", fullPath);
       await this._started.promise;
     }
 
-    const x = this._waitItems[fullPath];
-    assert(x !== undefined, `No local pkg build job started for pkg at ${fullPath}`);
+    const x = this._waitItems[fullPath]; // assert(x !== undefined, `No local pkg build job started for pkg at ${fullPath}`);
 
     if (x && x.promise) {
       logger.debug("waiting for build local item", fullPath, x);
@@ -4469,7 +4526,7 @@ class LocalPkgBuilder {
       logger.debug("build local item awaited", fullPath);
       return localBuildResult;
     } else {
-      logger.debug("status is false => no build local job for pkg at", fullPath, x);
+      logger.debug("no build local job for pkg at", fullPath, x);
     }
 
     return {};
@@ -5921,7 +5978,7 @@ const {
 const {
   getDepSection,
   makeDepStep
-} = __webpack_require__("./node_modules/.f/_/@fynpo/base/1.1.15-fynlocal_h/@fynpo/base/dist/index.js");
+} = __webpack_require__("./node_modules/.f/_/@fynpo/base/1.1.16-fynlocal_h/@fynpo/base/dist/index.js");
 
 const xaa = __webpack_require__("./lib/util/xaa.js");
 
@@ -6186,7 +6243,22 @@ class PkgDepResolver {
     const depthPkgs = Object.keys(depthInfo); // TODO: create test scenario for build-local
 
     if (this._options.buildLocal) {
-      const locals = depthPkgs.map(x => depthInfo[x].items.find(it => it.localType)).filter(x => x); // logger.info("adding depth pkgs", depthPkgs.join(", "), locals);
+      const locals = depthPkgs.map(x => depthInfo[x].items.find(it => it.localType)).filter(x => {
+        // if fynpo top level depends on a local package that's part of the monorepo, then
+        // we should not try to build the package while installing top level modules, because
+        // building a local package would likely require build tools that are typically
+        // installed as part of the top level packages
+        if (x && this._fyn.isTopLevelFynpoInstall()) {
+          const t1 = this._fyn._fynpo.graph.getPackageAtDir(Path.normalize(x.semverPath));
+
+          if (t1) {
+            logger.info("installing fynpo top level modules - skip build local for package at", x.semverPath);
+            return false;
+          }
+        }
+
+        return x;
+      }); // logger.info("adding depth pkgs", depthPkgs.join(", "), locals);
 
       if (locals.length > 0) {
         if (!this._localsByDepth) {
@@ -8979,7 +9051,7 @@ const {
 
 const {
   PackageRef
-} = __webpack_require__("./node_modules/.f/_/@fynpo/base/1.1.15-fynlocal_h/@fynpo/base/dist/index.js");
+} = __webpack_require__("./node_modules/.f/_/@fynpo/base/1.1.16-fynlocal_h/@fynpo/base/dist/index.js");
 
 const WATCH_TIME = 5000; // consider meta cache stale after this much time (30 minutes)
 
@@ -9165,7 +9237,7 @@ class PkgSrcManager {
     return readPkgJson(fullPath, true, true).then(json => {
       const publishUtilConfig = this.getPublishUtil(json, fullPath);
 
-      if (publishUtilConfig) {
+      if (publishUtilConfig && !publishUtilConfig.fynIgnore) {
         logger.debug(`processing local package.json at ${fullPath} with https://www.npmjs.com/package/publish-util prePackObj`);
         prePackObj(json, _objectSpread(_objectSpread({}, publishUtilConfig), {}, {
           silent: true
@@ -9246,6 +9318,7 @@ class PkgSrcManager {
 
 
     const pacoteRequest = () => {
+      logger.debug(`pacote.packument ${qItem.packumentUrl}`);
       return pacote.packument(pkgName, this.getPacoteOpts({
         "full-metadata": true,
         "fetch-retries": 3,
@@ -9466,6 +9539,7 @@ class PkgSrcManager {
         type: "meta",
         cacheKey,
         item,
+        packumentUrl,
         defer: createDefer()
       };
 
@@ -9631,7 +9705,23 @@ class PkgSrcManager {
       } else if (!(await central.allow(integrity))) {
         logger.info(`copying pkg ${dispId} in central store mode because it mutates in postinstall step.`);
       } else {
-        const hasCentral = await central.has(integrity);
+        let hasCentral = await central.has(integrity);
+
+        if (hasCentral) {
+          const valid = await central.validate(integrity);
+
+          if (!valid) {
+            const contentPath = await central.getContentPath(integrity);
+            logger.error(`Corrupted central store package detected
+  -- CORRUPTED CENTRAL STORE PACKAGE DETECTED, removing --
+  ID: ${verId}
+  integrity: ${integrity}
+  path: '${contentPath}'
+`);
+            await central.delete(integrity);
+            hasCentral = false;
+          }
+        }
 
         if (!hasCentral) {
           await central.storeTarStream(tarId, integrity, tarStream);
@@ -9997,7 +10087,7 @@ const {
 const {
   FynpoConfigManager,
   FynpoDepGraph
-} = __webpack_require__("./node_modules/.f/_/@fynpo/base/1.1.15-fynlocal_h/@fynpo/base/dist/index.js");
+} = __webpack_require__("./node_modules/.f/_/@fynpo/base/1.1.16-fynlocal_h/@fynpo/base/dist/index.js");
 /* eslint-disable no-magic-numbers, max-statements, no-empty, complexity, no-eval */
 
 
@@ -11518,7 +11608,7 @@ module.exports = __webpack_require__("./node_modules/.f/_/xaa/1.7.0/xaa/dist/xaa
 
 /***/ }),
 
-/***/ "./node_modules/.f/_/@fynpo/base/1.1.15-fynlocal_h/@fynpo/base/dist/caching.js":
+/***/ "./node_modules/.f/_/@fynpo/base/1.1.16-fynlocal_h/@fynpo/base/dist/caching.js":
 /***/ ((__unused_webpack_module, exports, __webpack_require__) => {
 
 "use strict";
@@ -11539,7 +11629,7 @@ const fs_1 = (0, tslib_1.__importDefault)(__webpack_require__("fs"));
 const path_1 = (0, tslib_1.__importDefault)(__webpack_require__("path"));
 const crypto_1 = (0, tslib_1.__importDefault)(__webpack_require__("crypto"));
 
-const minimatch_group_1 = __webpack_require__("./node_modules/.f/_/@fynpo/base/1.1.15-fynlocal_h/@fynpo/base/dist/minimatch-group.js");
+const minimatch_group_1 = __webpack_require__("./node_modules/.f/_/@fynpo/base/1.1.16-fynlocal_h/@fynpo/base/dist/minimatch-group.js");
 /**
  * create a sha256 hash for some data
  *
@@ -11807,7 +11897,7 @@ exports.processOutput = processOutput;
 
 /***/ }),
 
-/***/ "./node_modules/.f/_/@fynpo/base/1.1.15-fynlocal_h/@fynpo/base/dist/fynpo-config.js":
+/***/ "./node_modules/.f/_/@fynpo/base/1.1.16-fynlocal_h/@fynpo/base/dist/fynpo-config.js":
 /***/ ((__unused_webpack_module, exports, __webpack_require__) => {
 
 "use strict";
@@ -11957,7 +12047,7 @@ exports.FynpoConfigManager = FynpoConfigManager;
 
 /***/ }),
 
-/***/ "./node_modules/.f/_/@fynpo/base/1.1.15-fynlocal_h/@fynpo/base/dist/fynpo-dep-graph.js":
+/***/ "./node_modules/.f/_/@fynpo/base/1.1.16-fynlocal_h/@fynpo/base/dist/fynpo-dep-graph.js":
 /***/ ((__unused_webpack_module, exports, __webpack_require__) => {
 
 "use strict";
@@ -11987,9 +12077,9 @@ const minimatch_1 = (0, tslib_1.__importDefault)(__webpack_require__("./node_mod
 const lodash_1 = (0, tslib_1.__importDefault)(__webpack_require__("./node_modules/.f/_/lodash/4.17.21/lodash/lodash.min.js"));
 const semver_1 = (0, tslib_1.__importDefault)(__webpack_require__("./node_modules/.f/_/semver/7.3.5/semver/index.js"));
 
-const minimatch_group_1 = __webpack_require__("./node_modules/.f/_/@fynpo/base/1.1.15-fynlocal_h/@fynpo/base/dist/minimatch-group.js");
+const minimatch_group_1 = __webpack_require__("./node_modules/.f/_/@fynpo/base/1.1.16-fynlocal_h/@fynpo/base/dist/minimatch-group.js");
 
-const util_1 = __webpack_require__("./node_modules/.f/_/@fynpo/base/1.1.15-fynlocal_h/@fynpo/base/dist/util.js");
+const util_1 = __webpack_require__("./node_modules/.f/_/@fynpo/base/1.1.16-fynlocal_h/@fynpo/base/dist/util.js");
 
 const is_path_inside_1 = (0, tslib_1.__importDefault)(__webpack_require__("./node_modules/.f/_/is-path-inside/3.0.3/is-path-inside/index.js"));
 const DepSectionMap = {
@@ -12170,18 +12260,23 @@ class FynpoDepGraph {
     };
   }
   /**
-   * resolve packages from fynpo
+   * resolve packages from fynpo and resolve their dependency relations
+   *
+   * @param indirects - set true to resolve indirect deps also
    */
 
 
-  async resolve() {
+  async resolve(indirects = false) {
     if (lodash_1.default.isEmpty(this.packages.byName)) {
       await this.readPackages();
     }
 
-    this.resolveDirectDeps(); // we don't need indirect deps
-    // topo sort works with only direct deps
-    // this.resolveIndirectDeps();
+    this.resolveDirectDeps(); // default indirects to false because
+    // topo sort requires only direct deps to work
+
+    if (indirects) {
+      this.resolveIndirectDeps();
+    }
   }
   /**
    * update package depdencies map
@@ -12200,16 +12295,30 @@ class FynpoDepGraph {
    * @returns array of paths of package dep data
    */
 
+  /**
+   * Return array of dep data sorted by dependency topological order
+   *
+   * @param avoidCircs - don't consider circular deps if `true`
+   *
+   * @returns array of paths of package dep data
+   */
+
 
-  getTopoSortPackagePaths() {
+  getTopoSortPackagePaths(avoidCircs = false) {
     const depRecords = {};
-    const changed = []; //
+    const changed = []; // don't take optionalDependecies or peerDependencies into consideration
+    // TODO: also do not consider devDependencies
+    // - should calculate detail circular dep info and return that for logging
+    // - when fyn is about to do local build for a dep, it should check if there's
+    //   circular dep and avoid it.
+
+    const ignoreDepSections = ["opt", "peer"]; //
     // first start with packages that has zero local dependencies
     //
 
     for (const path in this.depMapByPath) {
       const depData = this.depMapByPath[path];
-      const count = Object.keys(depData.localDepsByPath).length;
+      const count = Object.keys(depData.localDepsByPath).filter(k => !ignoreDepSections.includes(depData.localDepsByPath[k].depSection) && (!avoidCircs || !this.depMapByPath[depData.localDepsByPath[k].path].hasCircular)).length;
       depRecords[path] = {
         depData,
         count
@@ -12247,9 +12356,21 @@ class FynpoDepGraph {
       }
     }
 
+    const circulars = Object.keys(depRecords).filter(path => depRecords[path].count > 0);
+    let avoidCircSorted = [];
+
+    if (!avoidCircs && circulars.length > 0) {
+      for (const p of circulars) {
+        this.depMapByPath[p].hasCircular = true;
+      }
+
+      avoidCircSorted = this.getTopoSortPackagePaths(true).sorted;
+    }
+
     return {
       sorted,
-      circulars: Object.keys(depRecords).filter(path => depRecords[path].count > 0)
+      avoidCircSorted,
+      circulars
     };
   }
   /**
@@ -12263,6 +12384,7 @@ class FynpoDepGraph {
     const topo = this.getTopoSortPackagePaths();
     return {
       sorted: topo.sorted.map(p => this.depMapByPath[p]),
+      avoidCircSorted: topo.avoidCircSorted.map(p => this.depMapByPath[p]),
       circulars: topo.circulars.map(p => this.depMapByPath[p])
     };
   }
@@ -12718,7 +12840,7 @@ exports.FynpoDepGraph = FynpoDepGraph;
 
 /***/ }),
 
-/***/ "./node_modules/.f/_/@fynpo/base/1.1.15-fynlocal_h/@fynpo/base/dist/index.js":
+/***/ "./node_modules/.f/_/@fynpo/base/1.1.16-fynlocal_h/@fynpo/base/dist/index.js":
 /***/ ((__unused_webpack_module, exports, __webpack_require__) => {
 
 "use strict";
@@ -12740,12 +12862,12 @@ const filter_scan_dir_1 = __webpack_require__("./node_modules/.f/_/filter-scan-d
 const minimatch_1 = (0, tslib_1.__importDefault)(__webpack_require__("./node_modules/.f/_/minimatch/3.0.4/minimatch/minimatch.js"));
 const lodash_1 = (0, tslib_1.__importDefault)(__webpack_require__("./node_modules/.f/_/lodash/4.17.21/lodash/lodash.min.js"));
 
-const minimatch_group_1 = __webpack_require__("./node_modules/.f/_/@fynpo/base/1.1.15-fynlocal_h/@fynpo/base/dist/minimatch-group.js");
+const minimatch_group_1 = __webpack_require__("./node_modules/.f/_/@fynpo/base/1.1.16-fynlocal_h/@fynpo/base/dist/minimatch-group.js");
 
-(0, tslib_1.__exportStar)(__webpack_require__("./node_modules/.f/_/@fynpo/base/1.1.15-fynlocal_h/@fynpo/base/dist/fynpo-dep-graph.js"), exports);
-(0, tslib_1.__exportStar)(__webpack_require__("./node_modules/.f/_/@fynpo/base/1.1.15-fynlocal_h/@fynpo/base/dist/fynpo-config.js"), exports);
-(0, tslib_1.__exportStar)(__webpack_require__("./node_modules/.f/_/@fynpo/base/1.1.15-fynlocal_h/@fynpo/base/dist/util.js"), exports);
-exports.caching = (0, tslib_1.__importStar)(__webpack_require__("./node_modules/.f/_/@fynpo/base/1.1.15-fynlocal_h/@fynpo/base/dist/caching.js"));
+(0, tslib_1.__exportStar)(__webpack_require__("./node_modules/.f/_/@fynpo/base/1.1.16-fynlocal_h/@fynpo/base/dist/fynpo-dep-graph.js"), exports);
+(0, tslib_1.__exportStar)(__webpack_require__("./node_modules/.f/_/@fynpo/base/1.1.16-fynlocal_h/@fynpo/base/dist/fynpo-config.js"), exports);
+(0, tslib_1.__exportStar)(__webpack_require__("./node_modules/.f/_/@fynpo/base/1.1.16-fynlocal_h/@fynpo/base/dist/util.js"), exports);
+exports.caching = (0, tslib_1.__importStar)(__webpack_require__("./node_modules/.f/_/@fynpo/base/1.1.16-fynlocal_h/@fynpo/base/dist/caching.js"));
 /**
  * Take an array of packages and figure out their dependencies on each other
  *
@@ -13001,7 +13123,7 @@ exports.makePkgDeps = makePkgDeps;
 
 /***/ }),
 
-/***/ "./node_modules/.f/_/@fynpo/base/1.1.15-fynlocal_h/@fynpo/base/dist/minimatch-group.js":
+/***/ "./node_modules/.f/_/@fynpo/base/1.1.16-fynlocal_h/@fynpo/base/dist/minimatch-group.js":
 /***/ ((__unused_webpack_module, exports, __webpack_require__) => {
 
 "use strict";
@@ -13140,7 +13262,7 @@ exports.unrollMmMatch = unrollMmMatch;
 
 /***/ }),
 
-/***/ "./node_modules/.f/_/@fynpo/base/1.1.15-fynlocal_h/@fynpo/base/dist/util.js":
+/***/ "./node_modules/.f/_/@fynpo/base/1.1.16-fynlocal_h/@fynpo/base/dist/util.js":
 /***/ ((__unused_webpack_module, exports, __webpack_require__) => {
 
 "use strict";
@@ -50236,7 +50358,7 @@ var gitHosts = module.exports = {
     'treepath': 'tree',
     'filetemplate': 'https://{auth@}raw.githubusercontent.com/{user}/{project}/{committish}/{path}',
     'bugstemplate': 'https://{domain}/{user}/{project}/issues',
-    'gittemplate': 'git://{auth@}{domain}/{user}/{project}.git{#committish}',
+    'gittemplate': 'https://{auth@}{domain}/{user}/{project}.git{#committish}',
     'tarballtemplate': 'https://codeload.{domain}/{user}/{project}/tar.gz/{committish}'
   },
   bitbucket: {
@@ -82853,18 +82975,19 @@ Fetcher.impl(fetchGit, {
 });
 
 function hostedManifest(spec, opts) {
+  // try https:// first before git://
   return BB.resolve(null).then(() => {
-    if (!spec.hosted.git()) {
-      throw new Error(`No git url for ${spec}`);
-    }
-
-    return plainManifest(spec.hosted.git(), spec, opts);
-  }).catch(err => {
     if (!spec.hosted.https()) {
       throw err;
     }
 
     return plainManifest(spec.hosted.https(), spec, opts);
+  }).catch(err => {
+    if (!spec.hosted.git()) {
+      throw new Error(`No git url for ${spec}`);
+    }
+
+    return plainManifest(spec.hosted.git(), spec, opts);
   }).catch(err => {
     if (!spec.hosted.sshurl()) {
       throw err;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "fyn",
-  "version": "1.1.39",
+  "version": "1.1.40",
   "description": "The package manager for fynpo, a zero setup monorepo manager",
   "main": "./bin/fyn.js",
   "homepage": "https://jchip.github.io/fynpo/",