@prosopo/config 0.2.11 → 0.2.13

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@prosopo/config",
-    "version": "0.2.11",
+    "version": "0.2.13",
     "description": "Prosopo config library",
     "main": "./dist/index.js",
     "type": "module",
@@ -17,13 +17,14 @@
     "author": "Prosopo Limited",
     "license": "Apache-2.0",
     "dependencies": {
-        "@prosopo/common": "0.2.11",
-        "@prosopo/types": "0.2.11",
-        "glob": "^10.0.0"
+        "@prosopo/common": "0.2.13",
+        "@prosopo/types": "0.2.13",
+        "glob": "^10.0.0",
+        "path-scurry": "^1.10.0"
     },
     "devDependencies": {
         "@polkadot/dev": "^0.76.11",
-        "@prosopo/util": "0.2.11",
+        "@prosopo/util": "0.2.13",
         "@rollup/plugin-alias": "^5.0.0",
         "@rollup/plugin-babel": "^6.0.3",
         "@rollup/plugin-commonjs": "^24.1.0",

package/src/dependencies.ts

@@ -0,0 +1,228 @@
+import { Glob } from 'glob'
+import { ProjectReference } from 'typescript'
+import { ProsopoEnvError, getLogger } from '@prosopo/common'
+import { at } from '@prosopo/util'
+import child_process from 'child_process'
+import fs from 'fs'
+import path from 'path'
+import util from 'util'
+
+const logger = getLogger(`Info`, `config.dependencies.js`)
+const exec = util.promisify(child_process.exec)
+// find a tScOnFiG.jSoN file
+const tsConfigRegex = /\/[A-Za-z.]*\.json$/
+const peerDepsRegex = /UNMET\sOPTIONAL\sDEPENDENCY\s+(@*[\w\-/.]+)@/
+const depsRegex = /\s+(@*[\w\-/.]+)@/
+async function getPackageDir(packageName: string): Promise<string> {
+    let pkg = packageName
+    if (packageName && !packageName.startsWith('@prosopo/')) {
+        pkg = `@prosopo/${packageName}`
+    }
+    const pkgCommand = `npm list ${pkg} -ap`
+    logger.info(`Running command ${pkgCommand}`)
+    // get package directory
+    const { stdout: packageDir, stderr } = await exec(pkgCommand)
+    if (stderr) {
+        throw new ProsopoEnvError(new Error(stderr))
+    }
+    return packageDir.trim()
+}
+
+/**
+ * Resolve the tsconfig path for a reference using the initial tsconfig path and the reference path. If the reference
+ * does not contain a tsconfig filename, `tsconfig.json` will be appended to the path.
+ * @param initialTsConfigPath
+ * @param reference
+ */
+function getReferenceTsConfigPath(initialTsConfigPath: string, reference: ProjectReference) {
+    // remove tsconfig.*.json from the path and get the path to the new directory via the reference path
+    let refTSConfigPath = path.resolve(initialTsConfigPath.replace(tsConfigRegex, ''), reference.path)
+    if (!refTSConfigPath.endsWith('.json')) {
+        refTSConfigPath = path.resolve(refTSConfigPath, 'tsconfig.json')
+    }
+    return refTSConfigPath
+}
+
+/**
+ * Get the tsconfig paths for a package
+ * @param tsConfigPath the tsconfig path to start with
+ * @param ignorePatterns the patterns to ignore
+ * @param tsConfigPaths the tsconfig paths to add to
+ * @param includeInitialTsConfig return the initial tsconfig path in the returned array
+ */
+export function getTsConfigs(
+    tsConfigPath: string,
+    ignorePatterns: RegExp[] = [],
+    tsConfigPaths: string[] = [],
+    includeInitialTsConfig = true
+): string[] {
+    let tsConfigs = [...tsConfigPaths]
+    //TODO use dynamic import with JSON assertion (TS complains that resolveJsonModule is not set)
+    const references = JSON.parse(fs.readFileSync(tsConfigPath).toString()).references
+    if (!tsConfigs.includes(tsConfigPath)) {
+        const ignore =
+            ignorePatterns && ignorePatterns.length > 0 ? new RegExp(`${ignorePatterns.join('|')}`) : undefined
+        if (includeInitialTsConfig) {
+            tsConfigs.push(tsConfigPath)
+        }
+
+        // ignore the packages we don't want to bundle
+        const filteredReferences = references.filter((reference: ProjectReference) =>
+            ignore ? !ignore.test(reference.path) : false
+        )
+        // for each reference, get the tsconfig paths - recursively calling this function
+        for (const reference of filteredReferences) {
+            // remove tsconfig.*.json from the path and get the path to the new directory via the reference path
+            const refTSConfigPath = getReferenceTsConfigPath(tsConfigPath, reference)
+
+            // take the reference TS config path (refTSConfigPath) and get the tsconfig paths for it (newTsConfigs),
+            // adding both to a distinct list, as there may be duplicates
+            const newTsConfigs = getTsConfigs(refTSConfigPath, ignorePatterns, tsConfigs)
+            if (newTsConfigs.length > 0) {
+                const distinctTsConfigPaths = new Set(tsConfigs.concat(newTsConfigs))
+                tsConfigs = [...distinctTsConfigPaths]
+            }
+        }
+    }
+    return tsConfigs
+}
+
+/**
+ * Get the workspace externals for a package
+ * @param tsConfigPath
+ * @param ignorePatterns
+ */
+export async function getExternalsFromReferences(
+    tsConfigPath: string,
+    ignorePatterns: RegExp[] = []
+): Promise<string[]> {
+    const tsConfigPaths = getTsConfigs(tsConfigPath, ignorePatterns, [], false)
+    logger.debug('tsConfigPaths', tsConfigPaths)
+    const promises: Promise<string>[] = []
+    for (const refTsConfigPath of tsConfigPaths) {
+        const packageJsonPath = path.resolve(refTsConfigPath.replace(tsConfigRegex, ''), 'package.json')
+        promises.push(
+            new Promise((resolve, reject) => {
+                // if package.json exists, read it and get the package name
+                fs.stat(packageJsonPath, (err) => {
+                    if (err) {
+                        reject(err)
+                    }
+                    fs.readFile(new URL(packageJsonPath, import.meta.url), function (err, buffer) {
+                        if (err) {
+                            reject(err)
+                        } else {
+                            const packageJson = JSON.parse(buffer.toString())
+                            const pkg = packageJson.name
+                            resolve(pkg)
+                        }
+                    })
+                })
+            })
+        )
+    }
+    const externals = await Promise.all(promises)
+    logger.debug('externals', externals)
+    return externals
+}
+
+/**
+ * Get the dependencies for a package
+ * @param packageName
+ * @param production
+ */
+export async function getDependencies(
+    packageName?: string,
+    production?: boolean
+): Promise<{ dependencies: string[]; optionalPeerDependencies: string[] }> {
+    let cmd = production ? 'npm ls -a --omit=dev' : 'npm ls -a'
+
+    if (packageName) {
+        const packageDir = await getPackageDir(packageName)
+        cmd = `cd ${packageDir.trim()} && ${cmd}`
+        logger.info(`Running command ${cmd} in ${packageDir}`)
+    }
+
+    const { stdout, stderr } = await exec(cmd)
+    if (stderr) {
+        throw new ProsopoEnvError(new Error(stderr))
+    }
+    const deps: string[] = []
+    const peerDeps: string[] = []
+    // for each line, check if there is an unmet optional dependency
+    stdout.split('\n').forEach((line) => {
+        if (line.includes('UNMET OPTIONAL DEPENDENCY')) {
+            //  │ │ │   ├── UNMET OPTIONAL DEPENDENCY bufferutil@^4.0.1
+            const parts = line.match(peerDepsRegex)
+            if (parts && parts.length > 1) {
+                peerDeps.push(at(parts, 1))
+            }
+        } else {
+            //  │ │ │ ├─┬ mongodb-memory-server-core@8.15.1
+            const parts = line.match(depsRegex)
+            if (parts && parts.length > 1) {
+                deps.push(at(parts, 1))
+            }
+        }
+    })
+    // dedupe and return deps and peer deps
+    return {
+        dependencies: deps.filter((x, i) => i === deps.indexOf(x)),
+        optionalPeerDependencies: peerDeps.filter((x, i) => i === peerDeps.indexOf(x)),
+    }
+}
+
+/**
+ * Filter out the dependencies we don't want
+ * @param deps
+ * @param filters
+ */
+export function filterDependencies(deps: string[], filters: string[]): { internal: string[]; external: string[] } {
+    const depsDeduped = deps.filter((x, i) => i === deps.indexOf(x))
+    const depsWithLength = depsDeduped.filter((dep) => dep.length > 0).sort()
+    const exclude = new RegExp(`${filters.join('|')}`)
+    // filter out the deps we don't want
+    const internal: string[] = []
+    const external: string[] = []
+    for (const dep of depsWithLength) {
+        if (exclude.test(dep)) {
+            external.push(dep)
+        } else {
+            internal.push(dep)
+        }
+    }
+    return { internal, external }
+}
+
+/** Takes an array of partial module directories, finds the full path, and returns an array containing the file paths
+ * of the files contained within the matching module directories [ filePath, filePath, ... ]
+ * @param startDir
+ * @param includePatterns
+ * @param excludePatterns
+ * @returns
+ * @example
+ * const includePatterns = ['kusama.js', 'westend.js']
+ * const excludePatterns = ['bytes.js']
+ * const startDir = path.resolve(__dirname, '../../node_modules/@polkadot')
+ * const files = getFilesInDirs(startDir, includePatterns, excludePatterns)
+ * console.log(files)
+ * // [ '/home/.../node_modules/@polkadot/types/interfaces/bytes/bytes.js',
+ * // '/home/.../node_modules/@polkadot/types/interfaces/bytes/bytes.d.ts']
+ * */
+export function getFilesInDirs(startDir: string, includePatterns: string[] = [], excludePatterns: string[] = []) {
+    const files: string[] = []
+    logger.info(`getFilesInDirs: ${startDir} excluding ${includePatterns} including ${excludePatterns}`)
+    const ignorePatterns = excludePatterns.map((pattern) => `${startDir}/**/${pattern}`)
+    includePatterns.forEach((searchPattern) => {
+        // get matching module directories
+        const globPattern = `${startDir}/**/${searchPattern}${searchPattern.indexOf('.') > -1 ? '' : '/*'}`
+        //log.info(`globPattern: ${globPattern}`)
+        const globResult = new Glob(globPattern, { recursive: true, ignore: ignorePatterns }).walkSync()
+        //log.info(`globResult: ${globResult}`)
+        for (const filePath of globResult) {
+            files.push(filePath)
+            //log.info(`ignoring ${filePath}`)
+        }
+    })
+    return files
+}

package/src/index.ts

@@ -0,0 +1,5 @@
+export * from './vite/index.js'
+export * from './dependencies.js'
+export * from './polkadot/index.js'
+export * from './provider/index.js'
+export * from './projectInfo.js'

package/src/polkadot/exclude.ts

@@ -0,0 +1,30 @@
+import { Alias } from 'vite'
+import { getLogger } from '@prosopo/common'
+import path from 'path'
+
+const log = getLogger(`Info`, `config.polkadot.exclude.js`)
+export function excludePolkadot() {
+    //'bytes.js'] //...interfacesToIgnore]
+    //const startDir = path.resolve(path.resolve(), '../../node_modules/@polkadot')
+    //log.info(`startDir: ${startDir}`)
+    //return getFilesInDirs(startDir, excludeFiles)
+    return ['./kusama.js', './westend.js', './cjs/bytes.js']
+}
+
+export function getAliases(dir: string): Alias[] {
+    const polkadotFilesToAlias = excludePolkadot()
+    const alias: Alias[] = []
+    // TODO Is there a way to determine this path by importing mock.js?
+    const mockFile = path.resolve(dir, '../config/dist/polkadot/mock.js')
+    polkadotFilesToAlias.forEach((file) => {
+        log.info(`resolving ${file} to mock.js`)
+        alias.push({ find: file, replacement: mockFile })
+    })
+    for (const pkg of ['api', 'api-contract', 'extension-dapp', 'types', 'util', 'util-crypto']) {
+        alias.push({
+            find: new RegExp(`/@polkadot/${pkg}$/`),
+            replacement: path.resolve(dir, `../../node_modules/@polkadot/${pkg}/bundle-polkadot-${pkg}.js`),
+        })
+    }
+    return alias
+}

package/src/polkadot/index.ts

@@ -0,0 +1 @@
+export { excludePolkadot, getAliases } from './exclude.js'

package/src/polkadot/mock.ts

@@ -0,0 +1,5 @@
+export const upgrades = []
+export const versioned = []
+export const statemint = []
+export const shell = []
+export default []

package/src/projectInfo.ts

@@ -0,0 +1,108 @@
+export const getContractNames = () => {
+    return ['captcha', 'common', 'proxy']
+}
+
+export type Paths = {
+    root: string
+    dev: string
+    demos: string
+    contract: string
+    protocol: string
+    packages: string
+    config: string
+    scripts: string
+    protocolCli: string
+    protocolDist: string
+    clientExample: string
+    clientExampleServer: string
+    clientBundleExample: string
+    dappExample: string
+    common: string
+    api: string
+    cli: string
+    util: string
+    database: string
+    datasets: string
+    datasetsFs: string
+    env: string
+    fileServer: string
+    procaptcha: string
+    procaptchaReact: string
+    procaptchaBundle: string
+    provider: string
+    server: string
+    typesDatabase: string
+    typesEnv: string
+    types: string
+    contractPackagesDir: string
+}
+
+export const getPaths = (): Paths => {
+    const root = new URL('../../..', import.meta.url).pathname.slice(0, -1)
+    const dev = `${root}/dev`
+    const packages = `${root}/packages`
+    const protocol = `${root}/protocol`
+    const demos = `${root}/demos`
+    const contract = `${packages}/contract`
+    const types = `${packages}/types`
+    const config = `${dev}/config`
+    const scripts = `${dev}/scripts`
+    const protocolCli = `${protocol}/dist/cli`
+    const protocolDist = `${protocol}/target/ink`
+    const clientExample = `${demos}/client-example`
+    const clientExampleServer = `${demos}/client-example-server`
+    const clientBundleExample = `${demos}/client-bundle-example`
+    const dappExample = `${demos}/dapp-example`
+    const common = `${packages}/common`
+    const api = `${packages}/api`
+    const cli = `${packages}/cli`
+    const util = `${packages}/util`
+    const database = `${packages}/database`
+    const datasets = `${packages}/datasets`
+    const datasetsFs = `${packages}/datasets-fs`
+    const env = `${packages}/env`
+    const fileServer = `${packages}/file-server`
+    const procaptcha = `${packages}/procaptcha`
+    const procaptchaReact = `${packages}/procaptcha-react`
+    const procaptchaBundle = `${packages}/procaptcha-bundle`
+    const provider = `${packages}/provider`
+    const server = `${packages}/server`
+    const typesDatabase = `${packages}/types-database`
+    const typesEnv = `${packages}/types-env`
+    const contractPackagesDir = `${root}/contracts`
+
+    return {
+        contractPackagesDir,
+        root,
+        dev,
+        demos,
+        contract,
+        protocol,
+        packages,
+        config,
+        scripts,
+        protocolCli,
+        protocolDist,
+        clientExample,
+        clientExampleServer,
+        clientBundleExample,
+        dappExample,
+        common,
+        api,
+        cli,
+        util,
+        database,
+        datasets,
+        datasetsFs,
+        env,
+        fileServer,
+        procaptcha,
+        procaptchaReact,
+        procaptchaBundle,
+        provider,
+        server,
+        typesDatabase,
+        typesEnv,
+        types,
+    }
+}

package/src/provider/index.ts

@@ -0,0 +1 @@
+export { default as getTestConfig } from './test.js'

package/src/provider/test.ts

@@ -0,0 +1,40 @@
+import {
+    DatabaseTypes,
+    EnvironmentTypesSchema,
+    NetworkNamesSchema,
+    NetworkPairTypeSchema,
+    ProsopoConfigSchema,
+} from '@prosopo/types'
+import { getLogLevel } from '@prosopo/common'
+
+const logLevel = getLogLevel()
+export default function getTestConfig() {
+    return ProsopoConfigSchema.parse({
+        logLevel,
+        defaultEnvironment: EnvironmentTypesSchema.Values.development,
+        defaultNetwork: NetworkNamesSchema.Values.development,
+        account: {
+            password: '',
+            address: '',
+        },
+        database: {
+            development: { type: DatabaseTypes.enum.mongoMemory, endpoint: '', dbname: 'prosopo_test', authSource: '' },
+        },
+        server: {
+            baseURL: 'http://localhost',
+            port: 9229,
+            fileServePaths: '[]',
+        },
+        networks: {
+            development: {
+                endpoint: 'ws://localhost:9944',
+                contract: {
+                    address: process.env.PROTOCOL_CONTRACT_ADDRESS || '',
+                    name: 'prosopo',
+                },
+                pairType: NetworkPairTypeSchema.parse('sr25519'),
+                ss58Format: 42,
+            },
+        },
+    })
+}

package/src/tests/paths.test.ts

@@ -0,0 +1,29 @@
+// Copyright 2021-2023 Prosopo (UK) Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+import { describe, expect, test } from 'vitest'
+import { flattenObj } from '@prosopo/util'
+import { getPaths } from '../projectInfo.js'
+import fs from 'fs'
+
+describe('paths', () => {
+    test('paths exist', () => {
+        const paths = getPaths()
+        const flat = flattenObj(paths)
+        for (const [key, value] of Object.entries(flat)) {
+            expect(value).to.not.be.undefined
+            expect(value).to.not.be.null
+            expect(fs.existsSync(String(value))).to.be.true
+        }
+    })
+})

package/src/vite/index.ts

@@ -0,0 +1,7 @@
+export { default as VitePluginCloseAndCopy } from './vite-plugin-close-and-copy.js'
+export { default as ViteTestConfig } from './vite.test.config.js'
+export { default as VitePluginSourceMapExclude } from './vite-plugin-sourcemap-exclude.js'
+export { default as ViteFrontendConfig } from './vite.frontend.config.js'
+export { default as ViteBackendConfig } from './vite.backend.config.js'
+export { default as VitePluginFixAbsoluteImports } from './vite-plugin-fix-absolute-imports.js'
+export { default as ViteCommonJSConfig } from './vite.commonjs.config.js'

package/src/vite/vite-plugin-close-and-copy.ts

@@ -0,0 +1,44 @@
+import { Plugin } from 'vite'
+import { getLogger } from '@prosopo/common'
+import fs from 'node:fs'
+import path from 'path'
+const log = getLogger(`Info`, `config.vite.vite-plugin-close.js`)
+export interface ClosePluginOptions {
+    srcDir: string
+    destDir: string
+    bundleName: string
+}
+/**
+ *   description: Closes Vite after the bundle has been build. Optionally copies the bundle to a different directory.
+ *   @param { ClosePluginOptions } options - The options object
+ **/
+export default function VitePluginCloseAndCopy(options?: ClosePluginOptions): Plugin {
+    const __dirname = path.resolve()
+    return {
+        name: 'close-plugin', // required, will show up in warnings and errors
+        buildStart() {
+            log.info('Bundle build started')
+        },
+        buildEnd(error) {
+            log.info('Build end')
+            if (error) {
+                console.log(error)
+                log.error(error)
+            }
+        },
+        closeBundle() {
+            if (options) {
+                for (const file of fs.readdirSync(path.resolve(__dirname, options.srcDir))) {
+                    if (file.startsWith(options.bundleName) && file.endsWith('js')) {
+                        const src = path.resolve(__dirname, options.srcDir, file)
+                        const dest = path.resolve(__dirname, options.destDir, file)
+                        fs.copyFileSync(src, dest)
+                        log.info(`Copied ${src} to ${dest}`)
+                    }
+                }
+            }
+            log.info('Bundle closed')
+            process.exit(0)
+        },
+    }
+}

package/src/vite/vite-plugin-fix-absolute-imports.ts

@@ -0,0 +1,24 @@
+import { builtinModules } from 'module'
+import type { Plugin } from 'vite'
+
+export default function VitePluginFixAbsoluteImports(): Plugin {
+    return {
+        name: 'fix-absolute-imports',
+        transform(code: string, id: string) {
+            for (const module of builtinModules) {
+                const moduleImportRegex = new RegExp(`from "(${module}(/.*)*/.*.js)";`, 'g')
+                // if matches, replace match with _module
+                const matches = code.match(moduleImportRegex)
+                if (matches) {
+                    for (const match of matches) {
+                        code = code.replace(match, `from "${module}";`)
+                    }
+                }
+            }
+            return {
+                code,
+            }
+        },
+        enforce: 'post',
+    }
+}

package/src/vite/vite-plugin-sourcemap-exclude.ts

@@ -0,0 +1,21 @@
+import type { Plugin } from 'vite'
+
+interface SourcemapExclude {
+    excludeNodeModules?: boolean
+}
+// Taken from https://github.com/vitejs/vite/issues/2433#issuecomment-1487472995 to prevent memory leaks
+export default function VitePluginSourcemapExclude(opts?: SourcemapExclude): Plugin {
+    return {
+        name: 'sourcemap-exclude',
+        transform(code: string, id: string) {
+            if (opts?.excludeNodeModules && id.includes('node_modules')) {
+                return {
+                    code,
+                    // https://github.com/rollup/rollup/blob/master/docs/plugin-development/index.md#source-code-transformations
+                    map: { mappings: '' },
+                }
+            }
+            return
+        },
+    }
+}