bundlebee 0.1.0 → 1.0.0

package/index.js

@@ -0,0 +1,400 @@
+const traverse = require('bare-module-traverse')
+const Bundle = require('bare-bundle')
+const Module = require('bare-module')
+const { fileURLToPath } = require('url-file-url')
+const fs = require('fs')
+const b4a = require('b4a')
+const ReadyResource = require('ready-resource')
+const Bee = require('hyperbee2')
+const c = require('compact-encoding')
+const { getEncoding } = require('./schema')
+
+const Entry = getEncoding('@bundlebee/entry')
+const Manifest = getEncoding('@bundlebee/manifest')
+const PeerDeps = getEncoding('@bundlebee/peer-deps')
+
+const MANIFEST_KEY_VALUE = '#manifest'
+const MANIFEST_KEY = b4a.from(MANIFEST_KEY_VALUE)
+const PEERDEPS_KEY_VALUE = '#peer-deps'
+const PEERDEPS_KEY = b4a.from(PEERDEPS_KEY_VALUE)
+
+// TODO
+// peer deps
+
+module.exports = class Bundlebee extends ReadyResource {
+  constructor(store, opts = {}) {
+    super()
+    this._bee = new Bee(store, { autoUpdate: true, ...opts })
+
+    this.ready().catch(noop)
+  }
+
+  static async require(store, ...files) {
+    const opts =
+      files.length &&
+      typeof files[files.length - 1] === 'object' &&
+      !('bundle' in files[files.length - 1])
+        ? files.pop()
+        : undefined
+
+    const b = new Bundlebee(store, opts)
+    const manifest = await b.manifest()
+
+    // skip requires an existing manifest
+    const skipExistingABIs = opts && !!opts.skipExistingABIs && !!manifest
+    const peerDependencies = opts?.peerDependencies
+
+    // Early exit
+    const lastAbi =
+      files.length && typeof files[files.length - 1] === 'object'
+        ? files[files.length - 1].abi || 0
+        : 0
+    if (skipExistingABIs && lastAbi <= manifest.abi) return b
+
+    const bundles = files.reduce((all, f) => {
+      const data = typeof f === 'object' ? f : { bundle: f, abi: 0 }
+      if (skipExistingABIs && data.abi <= manifest.abi) return all
+
+      all.push({
+        ...data,
+        bundle: Bundlebee.bundleFrom(data.bundle)
+      })
+
+      return all
+    }, [])
+
+    for (const bu of bundles) {
+      await b._addBundle(bu, peerDependencies)
+    }
+
+    return b
+  }
+
+  static bundleFrom(f) {
+    if (f.endsWith('.js')) {
+      const bundle = require(f)
+      return bundle
+    }
+
+    return Bundle.from(fs.readFileSync(f))
+  }
+
+  get core() {
+    return this._bee.core
+  }
+
+  head() {
+    return this._bee.head()
+  }
+
+  get key() {
+    return this._bee.core.key
+  }
+
+  get discoveryKey() {
+    return this._bee.core.discoveryKey
+  }
+
+  _open() {
+    return this._bee.ready()
+  }
+
+  async *createEntryStream(checkout) {
+    const b = checkout ? await this.checkout(checkout) : this._bee
+
+    for await (const data of b.createReadStream()) {
+      const id = data.key.toString()
+      if (id === MANIFEST_KEY_VALUE || id === PEERDEPS_KEY_VALUE) continue
+      const entry = c.decode(Entry, data.value)
+      yield { ...entry, id }
+    }
+  }
+
+  async manifest(checkout) {
+    if (!this.opened) await this.ready()
+
+    const b = checkout ? await this.checkout(checkout) : this._bee
+
+    const entry = await b.get(MANIFEST_KEY)
+    if (!entry) return null
+
+    return c.decode(Manifest, entry.value)
+  }
+
+  async peerDependencies(checkout) {
+    if (!this.opened) await this.ready()
+
+    const b = checkout ? await this.checkout(checkout) : this._bee
+
+    const entry = await b.get(b4a.from('#peer-deps'))
+    if (!entry) return null
+
+    const { packages } = c.decode(PeerDeps, entry.value)
+
+    return new Set(packages)
+  }
+
+  async get(key, checkout) {
+    if (!this.opened) await this.ready()
+
+    const b = checkout ? await this.checkout(checkout) : this._bee
+
+    const entry = await b.get(b4a.from(key))
+    if (!entry) return null
+
+    return c.decode(Entry, entry.value)
+  }
+
+  async findABI(abi) {
+    for await (const d of this._bee.createChangesStream()) {
+      let record = null
+      for (const b of d.batch) {
+        if (!b.keys) continue
+        const r = b.keys.find((k) => k.key.toString() === MANIFEST_KEY_VALUE)
+        if (!r) continue
+        record = r
+      }
+      if (!record) continue
+
+      const manifest = c.decode(Manifest, record.value)
+      if (manifest.abi !== abi) continue
+
+      return d.head
+    }
+  }
+
+  async checkout(checkout) {
+    if (!this.opened) await this.ready()
+
+    return this._bee.checkout(checkout)
+  }
+
+  async load(root, entry, checkout, { cache = require.cache, skipModules = true } = {}) {
+    if (!this.opened) await this.ready()
+    if (!(await this.get(entry, checkout))) throw new Error(`${entry} not found`)
+
+    const b = await this.checkout(checkout)
+    const loadedData = new Map()
+
+    const protocol = new Module.Protocol({
+      exists(url) {
+        return true
+      },
+      read(url) {
+        const p = url.pathname
+        return loadedData.get(p)
+      }
+    })
+
+    const bundleCache = Object.create(null)
+    const allResolutions = Object.create(null)
+
+    const peerDeps = await this.peerDependencies(checkout)
+
+    for await (const data of b.createReadStream()) {
+      const id = data.key.toString()
+      if (id === MANIFEST_KEY_VALUE || id === PEERDEPS_KEY_VALUE) continue
+
+      const { resolutions, source } = c.decode(Entry, data.value)
+
+      loadedData.set(id, source)
+
+      const m = {}
+      for (const [k, v] of Object.entries(resolutions)) {
+        const skip = peerDeps && peerDeps.has(k)
+        const nm =
+          (v.startsWith('/node_modules') && skipModules) || skip ? findModule(cache, v, root) : null
+        if (nm) {
+          m[k] = 'bundle://host' + v
+          bundleCache[m[k]] = nm
+        } else {
+          m[k] = 'bundle://layer' + v
+        }
+      }
+      allResolutions['bundle://layer' + id] = m
+    }
+
+    return Module.load(new URL(entry, 'bundle://layer/'), {
+      protocol,
+      resolutions: allResolutions,
+      cache: bundleCache
+    })
+  }
+
+  async add(root, entry, { skipModules = true, peerDependencies } = {}) {
+    if (!this.opened) await this.ready()
+    if (!root.pathname.endsWith('/')) root = new URL('./', root)
+    if (peerDependencies) peerDependencies = new Set(peerDependencies)
+
+    const nodeModules = new URL('./node_modules', root)
+    const bundle = new Bundle()
+
+    const resolutions = {}
+
+    for await (const dependency of traverse(
+      new URL(entry, root),
+      { resolve: traverse.resolve.bare },
+      readModule,
+      listPrefix
+    )) {
+      if (dependency.url.href.startsWith(nodeModules.href)) {
+        if (skipModules) continue
+        if (peerDependencies) {
+          const moduleName = dependency.url.pathname
+            .replace(root.pathname + 'node_modules/', '')
+            .split('/')[0]
+          if (peerDependencies.has(moduleName)) continue
+        }
+      }
+
+      const p = dependency.url.pathname.replace(root.pathname, '/')
+      const imps = {}
+      for (const [k, v] of Object.entries(dependency.imports)) {
+        imps[k] = new URL(v).pathname.replace(root.pathname, '/')
+      }
+
+      const existing = await this.get(p)
+      if (
+        existing &&
+        b4a.equals(existing.source, dependency.source) &&
+        sameImports(existing.resolutions, imps)
+      ) {
+        continue
+      }
+
+      bundle.write(p, dependency.source)
+      resolutions[p] = imps
+    }
+
+    bundle.resolutions = resolutions
+
+    await this._addBundle({ bundle }, peerDependencies)
+
+    return bundle
+  }
+
+  async _addBundle(data, peerDependencies) {
+    if (!this.opened) await this.ready()
+
+    let nextAbi = data.abi
+    const previousManifest = await this.manifest()
+    if (nextAbi && previousManifest && nextAbi <= previousManifest.abi) {
+      throw new Error(`ABI ${nextAbi} <= to current ABI ${previousManifest.abi}`)
+    } else if (!nextAbi) {
+      nextAbi = previousManifest ? previousManifest.abi + 1 : 1
+    }
+
+    const w = this._bee.write()
+    for (const f in data.bundle.files) {
+      // TODO: make a schema for resolutions value
+      // source + resolutions map
+
+      w.tryPut(
+        b4a.from(f),
+        c.encode(Entry, {
+          source: data.bundle.files[f].read(),
+          resolutions: data.bundle.resolutions[f]
+        })
+      )
+    }
+
+    if (peerDependencies) {
+      w.tryPut(
+        b4a.from(PEERDEPS_KEY),
+        c.encode(PeerDeps, {
+          packages: [...peerDependencies]
+        })
+      )
+    }
+
+    w.tryPut(
+      b4a.from(MANIFEST_KEY),
+      c.encode(Manifest, {
+        abi: nextAbi
+      })
+    )
+
+    await w.flush()
+  }
+}
+
+function noop() {}
+
+function sameImports(a, b) {
+  const x = Object.keys(a)
+  const y = Object.keys(b)
+
+  if (x.length !== y.length) return false
+
+  for (let i = 0; i < x.length; i++) {
+    if (a[x[i]] !== b[x[i]]) return false
+  }
+
+  return true
+}
+
+async function readModule(url) {
+  return new Promise((resolve) => {
+    fs.readFile(fileURLToPath(url), (err, data) => {
+      resolve(err ? null : data)
+    })
+  })
+}
+
+async function openDir(url) {
+  return new Promise((resolve, reject) => {
+    fs.opendir(fileURLToPath(url), (err, dir) => {
+      err ? reject(err) : resolve(dir)
+    })
+  })
+}
+
+async function isFile(url) {
+  return new Promise((resolve) => {
+    fs.stat(fileURLToPath(url), (err, stat) => {
+      resolve(err === null && stat.isFile())
+    })
+  })
+}
+
+async function isDir(url) {
+  return new Promise((resolve) => {
+    fs.stat(fileURLToPath(url), (err, stat) => {
+      resolve(err === null && stat.isDirectory())
+    })
+  })
+}
+
+async function* listPrefix(url) {
+  if (await isFile(url)) return yield url
+
+  if (url.pathname[url.pathname.length - 1] !== '/') {
+    url.pathname += '/'
+  }
+
+  if (await isDir(url)) {
+    for await (const entry of await openDir(url)) {
+      if (entry.isDirectory()) {
+        yield* listPrefix(new URL(entry.name, url))
+      } else {
+        yield new URL(entry.name, url)
+      }
+    }
+  }
+}
+
+function findModule(cache, v, root) {
+  let s = '.'
+  let prev = null
+
+  while (true) {
+    const cand = new URL(s + v, root).href
+    s += '/..'
+    if (prev === cand) break
+    prev = cand
+    const nm = cache[cand]
+    if (nm) return nm
+  }
+
+  throw new Error(`failed to find module: ${v}`)
+}

package/package.json

@@ -1,39 +1,62 @@
 {
   "name": "bundlebee",
-  "version": "0.1.0",
-  "description": "Create bundles with a simple interface independant of the bundler",
-  "main": "dist/bundlebee.js",
-  "jsnext:main": "src/index.js",
+  "version": "1.0.0",
+  "description": "bundlebee",
+  "main": "index.js",
+  "imports": {
+    "fs": {
+      "bare": "bare-fs",
+      "default": "fs"
+    }
+  },
+  "exports": {
+    "./package": "./package.json",
+    ".": {
+      "types": "./index.d.ts",
+      "default": "./index.js"
+    }
+  },
+  "files": [
+    "package.json",
+    "index.js",
+    "index.d.ts",
+    "schema"
+  ],
+  "devDependencies": {
+    "brittle": "^3.19.0",
+    "corestore": "^7.8.0",
+    "hyperdht": "^6.29.0",
+    "hyperswarm": "^4.16.0",
+    "lunte": "^1.2.0",
+    "prettier": "^3.6.2",
+    "prettier-config-holepunch": "^2.0.0",
+    "test-tmp": "^1.4.0"
+  },
   "scripts": {
-    "build": "rollup-babel-lib-bundler src/index.js",
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "format": "prettier . --write",
+    "test": "prettier . --check && lunte && brittle-bare test/index.js",
+    "lint": "lunte"
   },
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/frostney/bundlebee.git"
+    "url": "git+https://github.com/holepunchto/bundlebee.git"
   },
-  "keywords": [
-    "bundle",
-    "bundler",
-    "cmake",
-    "make",
-    "config"
-  ],
-  "files": [
-    "dist",
-    "src"
-  ],
-  "author": "Johannes Stein",
-  "license": "MIT",
+  "author": "Holepunch Inc",
+  "license": "Apache-2.0",
   "bugs": {
-    "url": "https://github.com/frostney/bundlebee/issues"
+    "url": "https://github.com/holepunchto/bundlebee/issues"
   },
-  "homepage": "https://github.com/frostney/bundlebee#readme",
+  "homepage": "https://github.com/holepunchto/bundlebee",
   "dependencies": {
-    "glob-to-regexp": "^0.1.0"
-  },
-  "devDependencies": {
-    "babel-preset-es2015-loose-rollup": "^7.0.0",
-    "rollup-babel-lib-bundler": "^2.1.1"
+    "b4a": "^1.7.4",
+    "bare-bundle": "^1.10.0",
+    "bare-fs": "^4.5.4",
+    "bare-module": "^6.1.3",
+    "bare-module-traverse": "^2.0.1",
+    "compact-encoding": "^2.19.0",
+    "hyperbee2": "^2.7.1",
+    "hyperschema": "^1.20.1",
+    "ready-resource": "^1.2.0",
+    "url-file-url": "^1.0.5"
   }
 }

package/schema/index.js

@@ -0,0 +1,140 @@
+// This file is autogenerated by the hyperschema compiler
+// Schema Version: 1
+/* eslint-disable camelcase */
+/* eslint-disable quotes */
+/* eslint-disable space-before-function-paren */
+
+const { c } = require('hyperschema/runtime')
+
+const VERSION = 1
+
+// eslint-disable-next-line no-unused-vars
+let version = VERSION
+
+// @bundlebee/resolutions
+const encoding0 = c.record(c.string, c.string)
+
+// @bundlebee/entry
+const encoding1 = {
+  preencode(state, m) {
+    c.buffer.preencode(state, m.source)
+    encoding0.preencode(state, m.resolutions)
+  },
+  encode(state, m) {
+    c.buffer.encode(state, m.source)
+    encoding0.encode(state, m.resolutions)
+  },
+  decode(state) {
+    const r0 = c.buffer.decode(state)
+    const r1 = encoding0.decode(state)
+
+    return {
+      source: r0,
+      resolutions: r1
+    }
+  }
+}
+
+// @bundlebee/manifest
+const encoding2 = {
+  preencode(state, m) {
+    c.uint.preencode(state, m.abi)
+  },
+  encode(state, m) {
+    c.uint.encode(state, m.abi)
+  },
+  decode(state) {
+    const r0 = c.uint.decode(state)
+
+    return {
+      abi: r0
+    }
+  }
+}
+
+// @bundlebee/peer-deps.packages
+const encoding3_0 = c.array(c.string)
+
+// @bundlebee/peer-deps
+const encoding3 = {
+  preencode(state, m) {
+    encoding3_0.preencode(state, m.packages)
+  },
+  encode(state, m) {
+    encoding3_0.encode(state, m.packages)
+  },
+  decode(state) {
+    const r0 = encoding3_0.decode(state)
+
+    return {
+      packages: r0
+    }
+  }
+}
+
+function setVersion(v) {
+  version = v
+}
+
+function encode(name, value, v = VERSION) {
+  version = v
+  return c.encode(getEncoding(name), value)
+}
+
+function decode(name, buffer, v = VERSION) {
+  version = v
+  return c.decode(getEncoding(name), buffer)
+}
+
+function getEnum(name) {
+  switch (name) {
+    default:
+      throw new Error('Enum not found ' + name)
+  }
+}
+
+function getEncoding(name) {
+  switch (name) {
+    case '@bundlebee/resolutions':
+      return encoding0
+    case '@bundlebee/entry':
+      return encoding1
+    case '@bundlebee/manifest':
+      return encoding2
+    case '@bundlebee/peer-deps':
+      return encoding3
+    default:
+      throw new Error('Encoder not found ' + name)
+  }
+}
+
+function getStruct(name, v = VERSION) {
+  const enc = getEncoding(name)
+  return {
+    preencode(state, m) {
+      version = v
+      enc.preencode(state, m)
+    },
+    encode(state, m) {
+      version = v
+      enc.encode(state, m)
+    },
+    decode(state) {
+      version = v
+      return enc.decode(state)
+    }
+  }
+}
+
+const resolveStruct = getStruct // compat
+
+module.exports = {
+  resolveStruct,
+  getStruct,
+  getEnum,
+  getEncoding,
+  encode,
+  decode,
+  setVersion,
+  version
+}

package/schema/schema.json

@@ -0,0 +1,61 @@
+{
+  "version": 1,
+  "schema": [
+    {
+      "name": "resolutions",
+      "namespace": "bundlebee",
+      "record": true,
+      "key": "string",
+      "value": "string"
+    },
+    {
+      "name": "entry",
+      "namespace": "bundlebee",
+      "compact": false,
+      "flagsPosition": -1,
+      "fields": [
+        {
+          "name": "source",
+          "required": true,
+          "type": "buffer",
+          "version": 1
+        },
+        {
+          "name": "resolutions",
+          "required": true,
+          "type": "@bundlebee/resolutions",
+          "version": 1
+        }
+      ]
+    },
+    {
+      "name": "manifest",
+      "namespace": "bundlebee",
+      "compact": false,
+      "flagsPosition": -1,
+      "fields": [
+        {
+          "name": "abi",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        }
+      ]
+    },
+    {
+      "name": "peer-deps",
+      "namespace": "bundlebee",
+      "compact": false,
+      "flagsPosition": -1,
+      "fields": [
+        {
+          "name": "packages",
+          "required": true,
+          "array": true,
+          "type": "string",
+          "version": 1
+        }
+      ]
+    }
+  ]
+}