@budibase/cli 0.0.1

package/src/analytics/index.ts

@@ -0,0 +1,60 @@
+import { Command } from "../structures/Command"
+import { CommandWord } from "../constants"
+import { success, error } from "../utils"
+import { AnalyticsClient } from "./Client"
+
+const client = new AnalyticsClient()
+
+async function optOut() {
+  try {
+    // opt them out
+    client.disable()
+    console.log(
+      success(
+        "Successfully opted out of Budibase analytics. You can opt in at any time by running 'budi analytics opt-in'"
+      )
+    )
+  } catch (err: any) {
+    console.log(
+      error(
+        `Error opting out of Budibase analytics. Please try again later - ${err}`
+      )
+    )
+  }
+}
+
+async function optIn() {
+  try {
+    // opt them in
+    client.enable()
+    console.log(
+      success(
+        "Successfully opted in to Budibase analytics. Thank you for helping us make Budibase better!"
+      )
+    )
+  } catch (err) {
+    console.log(
+      error("Error opting in to Budibase analytics. Please try again later.")
+    )
+  }
+}
+
+async function status() {
+  try {
+    console.log(success(`Budibase analytics ${client.status()}`))
+  } catch (err) {
+    console.log(
+      error("Error fetching analytics status. Please try again later.")
+    )
+  }
+}
+
+export default new Command(`${CommandWord.ANALYTICS}`)
+  .addHelp("Control the analytics you send to Budibase.")
+  .addSubOption("--optin", "Opt in to sending analytics to Budibase", optIn)
+  .addSubOption("--optout", "Opt out of sending analytics to Budibase.", optOut)
+  .addSubOption(
+    "--status",
+    "Check whether you are currently opted in to Budibase analytics.",
+    status
+  )

package/src/backups/index.ts

@@ -0,0 +1,130 @@
+import { Command } from "../structures/Command"
+import { CommandWord } from "../constants"
+import fs from "fs"
+import { join } from "path"
+import { getAllDbs } from "../core/db"
+import { progressBar, httpCall } from "../utils"
+import {
+  TEMP_DIR,
+  COUCH_DIR,
+  MINIO_DIR,
+  getConfig,
+  replication,
+  getPouches,
+} from "./utils"
+import { exportObjects, importObjects } from "./objectStore"
+const tar = require("tar")
+
+type BackupOpts = { env?: string; import?: string; export?: string }
+
+async function exportBackup(opts: BackupOpts) {
+  const envFile = opts.env || undefined
+  let filename = opts["export"] || (opts as string)
+  if (typeof filename !== "string") {
+    filename = `backup-${new Date().toISOString()}.tar.gz`
+  }
+  const config = await getConfig(envFile)
+  const dbList = (await getAllDbs(config["COUCH_DB_URL"])) as string[]
+  const { Remote, Local } = getPouches(config)
+  if (fs.existsSync(TEMP_DIR)) {
+    fs.rmSync(TEMP_DIR, { recursive: true })
+  }
+  const couchDir = join(TEMP_DIR, COUCH_DIR)
+  fs.mkdirSync(TEMP_DIR)
+  fs.mkdirSync(couchDir)
+  console.log("CouchDB Export")
+  const bar = progressBar(dbList.length)
+  let count = 0
+  for (let db of dbList) {
+    bar.update(++count)
+    const remote = new Remote(db)
+    const local = new Local(join(TEMP_DIR, COUCH_DIR, db))
+    await replication(remote, local)
+  }
+  bar.stop()
+  console.log("S3 Export")
+  await exportObjects()
+  tar.create(
+    {
+      sync: true,
+      gzip: true,
+      file: filename,
+      cwd: join(TEMP_DIR),
+    },
+    [COUCH_DIR, MINIO_DIR]
+  )
+  fs.rmSync(TEMP_DIR, { recursive: true })
+  console.log(`Generated export file - ${filename}`)
+}
+
+async function importBackup(opts: BackupOpts) {
+  const envFile = opts.env || undefined
+  const filename = opts["import"] || (opts as string)
+  const config = await getConfig(envFile)
+  if (!filename || !fs.existsSync(filename)) {
+    console.error("Cannot import without specifying a valid file to import")
+    process.exit(-1)
+  }
+  if (fs.existsSync(TEMP_DIR)) {
+    fs.rmSync(TEMP_DIR, { recursive: true })
+  }
+  fs.mkdirSync(TEMP_DIR)
+  tar.extract({
+    sync: true,
+    cwd: join(TEMP_DIR),
+    file: filename,
+  })
+  const { Remote, Local } = getPouches(config)
+  const dbList = fs.readdirSync(join(TEMP_DIR, COUCH_DIR))
+  console.log("CouchDB Import")
+  const bar = progressBar(dbList.length)
+  let count = 0
+  for (let db of dbList) {
+    bar.update(++count)
+    const remote = new Remote(db)
+    const local = new Local(join(TEMP_DIR, COUCH_DIR, db))
+    await replication(local, remote)
+  }
+  bar.stop()
+  console.log("MinIO Import")
+  await importObjects()
+  // finish by letting the system know that a restore has occurred
+  try {
+    await httpCall(
+      `http://localhost:${config.MAIN_PORT}/api/system/restored`,
+      "POST"
+    )
+  } catch (err) {
+    // ignore error - it will be an older system
+  }
+  console.log("Import complete")
+  fs.rmSync(TEMP_DIR, { recursive: true })
+}
+
+async function pickOne(opts: BackupOpts) {
+  if (opts["import"]) {
+    return importBackup(opts)
+  } else if (opts["export"]) {
+    return exportBackup(opts)
+  }
+}
+
+export default new Command(`${CommandWord.BACKUPS}`)
+  .addHelp(
+    "Allows building backups of Budibase, as well as importing a backup to a new instance."
+  )
+  .addSubOption(
+    "--export [filename]",
+    "Export a backup from an existing Budibase installation.",
+    exportBackup
+  )
+  .addSubOption(
+    "--import [filename]",
+    "Import a backup to a new Budibase installation.",
+    importBackup
+  )
+  .addSubOption(
+    "--env [envFile]",
+    "Provide an environment variable file to configure the CLI.",
+    pickOne
+  )

package/src/backups/objectStore.ts

@@ -0,0 +1,69 @@
+import { objectStore } from "@budibase/backend-core"
+import fs from "fs"
+import { join } from "path"
+import { TEMP_DIR, MINIO_DIR } from "./utils"
+import { progressBar } from "../utils"
+const {
+  ObjectStoreBuckets,
+  ObjectStore,
+  retrieve,
+  uploadDirectory,
+  makeSureBucketExists,
+} = objectStore
+
+const bucketList = Object.values(ObjectStoreBuckets)
+
+export async function exportObjects() {
+  const path = join(TEMP_DIR, MINIO_DIR)
+  fs.mkdirSync(path)
+  let fullList: any[] = []
+  let errorCount = 0
+  for (let bucket of bucketList) {
+    const client = ObjectStore(bucket)
+    try {
+      await client.headBucket().promise()
+    } catch (err) {
+      errorCount++
+      continue
+    }
+    const list = (await client.listObjectsV2().promise()) as { Contents: any[] }
+    fullList = fullList.concat(list.Contents.map(el => ({ ...el, bucket })))
+  }
+  if (errorCount === bucketList.length) {
+    throw new Error("Unable to access MinIO/S3 - check environment config.")
+  }
+  const bar = progressBar(fullList.length)
+  let count = 0
+  for (let object of fullList) {
+    const filename = object.Key
+    const data = await retrieve(object.bucket, filename)
+    const possiblePath = filename.split("/")
+    if (possiblePath.length > 1) {
+      const dirs = possiblePath.slice(0, possiblePath.length - 1)
+      fs.mkdirSync(join(path, object.bucket, ...dirs), { recursive: true })
+    }
+    fs.writeFileSync(join(path, object.bucket, ...possiblePath), data)
+    bar.update(++count)
+  }
+  bar.stop()
+}
+
+export async function importObjects() {
+  const path = join(TEMP_DIR, MINIO_DIR)
+  const buckets = fs.readdirSync(path)
+  let total = 0
+  buckets.forEach(bucket => {
+    const files = fs.readdirSync(join(path, bucket))
+    total += files.length
+  })
+  const bar = progressBar(total)
+  let count = 0
+  for (let bucket of buckets) {
+    const client = ObjectStore(bucket)
+    await makeSureBucketExists(client, bucket)
+    const files = await uploadDirectory(bucket, join(path, bucket), "/")
+    count += files.length
+    bar.update(count)
+  }
+  bar.stop()
+}

package/src/backups/utils.ts

@@ -0,0 +1,99 @@
+import dotenv from "dotenv"
+import fs from "fs"
+import { string } from "../questions"
+import { getPouch } from "../core/db"
+import { env as environment } from "@budibase/backend-core"
+import PouchDB from "pouchdb"
+
+export const TEMP_DIR = ".temp"
+export const COUCH_DIR = "couchdb"
+export const MINIO_DIR = "minio"
+
+const REQUIRED = [
+  { value: "MAIN_PORT", default: "10000" },
+  {
+    value: "COUCH_DB_URL",
+    default: "http://budibase:budibase@localhost:10000/db/",
+  },
+  { value: "MINIO_URL", default: "http://localhost:10000" },
+  { value: "MINIO_ACCESS_KEY" },
+  { value: "MINIO_SECRET_KEY" },
+]
+
+export function checkURLs(config: Record<string, string>) {
+  const mainPort = config["MAIN_PORT"],
+    username = config["COUCH_DB_USER"],
+    password = config["COUCH_DB_PASSWORD"]
+  if (!config["COUCH_DB_URL"] && mainPort && username && password) {
+    config[
+      "COUCH_DB_URL"
+    ] = `http://${username}:${password}@localhost:${mainPort}/db/`
+  }
+  if (!config["MINIO_URL"]) {
+    config["MINIO_URL"] = `http://localhost:${mainPort}/`
+  }
+  return config
+}
+
+export async function askQuestions() {
+  console.log(
+    "*** NOTE: use a .env file to load these parameters repeatedly ***"
+  )
+  let config: Record<string, string> = {}
+  for (let property of REQUIRED) {
+    config[property.value] = await string(property.value, property.default)
+  }
+  return config
+}
+
+export function loadEnvironment(path: string) {
+  if (!fs.existsSync(path)) {
+    throw "Unable to file specified .env file"
+  }
+  const env = fs.readFileSync(path, "utf8")
+  const config = checkURLs(dotenv.parse(env))
+  for (let required of REQUIRED) {
+    if (!config[required.value]) {
+      throw `Cannot find "${required.value}" property in .env file`
+    }
+  }
+  return config
+}
+
+// true is the default value passed by commander
+export async function getConfig(envFile: boolean | string = true) {
+  let config
+  if (envFile !== true) {
+    config = loadEnvironment(envFile as string)
+  } else {
+    config = await askQuestions()
+  }
+  // fill out environment
+  for (let key of Object.keys(config)) {
+    environment._set(key, config[key])
+  }
+  return config
+}
+
+export async function replication(
+  from: PouchDB.Database,
+  to: PouchDB.Database
+) {
+  const pouch = getPouch()
+  try {
+    await pouch.replicate(from, to, {
+      batch_size: 1000,
+      batches_limit: 5,
+      // @ts-ignore
+      style: "main_only",
+    })
+  } catch (err) {
+    throw new Error(`Replication failed - ${JSON.stringify(err)}`)
+  }
+}
+
+export function getPouches(config: Record<string, string>) {
+  const Remote = getPouch(config["COUCH_DB_URL"])
+  const Local = getPouch()
+  return { Remote, Local }
+}

package/src/constants.ts

@@ -0,0 +1,4 @@
+export { CommandWord, InitType, AnalyticsEvent } from "@budibase/types"
+
+export const POSTHOG_TOKEN = "phc_yGOn4i7jWKaCTapdGR6lfA4AvmuEQ2ijn5zAVSFYPlS"
+export const GENERATED_USER_EMAIL = "admin@admin.com"

package/src/core/db.ts

@@ -0,0 +1,39 @@
+import PouchDB from "pouchdb"
+import { checkSlashesInUrl } from "../utils"
+import fetch from "node-fetch"
+
+/**
+ * Fully qualified URL including username and password, or nothing for local
+ */
+export function getPouch(url?: string) {
+  let POUCH_DB_DEFAULTS
+  if (!url) {
+    POUCH_DB_DEFAULTS = {
+      prefix: undefined,
+      adapter: "leveldb",
+    }
+  } else {
+    POUCH_DB_DEFAULTS = {
+      prefix: url,
+    }
+  }
+  const replicationStream = require("pouchdb-replication-stream")
+  PouchDB.plugin(replicationStream.plugin)
+  // @ts-ignore
+  PouchDB.adapter("writableStream", replicationStream.adapters.writableStream)
+  return PouchDB.defaults(POUCH_DB_DEFAULTS) as PouchDB.Static
+}
+
+export async function getAllDbs(url: string) {
+  const response = await fetch(
+    checkSlashesInUrl(encodeURI(`${url}/_all_dbs`)),
+    {
+      method: "GET",
+    }
+  )
+  if (response.status === 200) {
+    return await response.json()
+  } else {
+    throw "Cannot connect to CouchDB instance"
+  }
+}

package/src/environment.ts

@@ -0,0 +1,3 @@
+process.env.NO_JS = "1"
+process.env.JS_BCRYPT = "1"
+process.env.DISABLE_JWT_WARNING = "1"

package/src/events.ts

@@ -0,0 +1,11 @@
+import { AnalyticsClient } from "./analytics/Client"
+
+const client = new AnalyticsClient()
+
+export function captureEvent(event: string, properties: any) {
+  client.capture({
+    distinctId: "cli",
+    event,
+    properties,
+  })
+}

package/src/exec.ts

@@ -0,0 +1,27 @@
+import util from "util"
+const runCommand = util.promisify(require("child_process").exec)
+
+export async function exec(command: string, dir = "./") {
+  const { stdout } = await runCommand(command, { cwd: dir })
+  return stdout
+}
+
+export async function utilityInstalled(utilName: string) {
+  try {
+    await exec(`${utilName} --version`)
+    return true
+  } catch (err) {
+    return false
+  }
+}
+
+export async function runPkgCommand(command: string, dir = "./") {
+  const yarn = await exports.utilityInstalled("yarn")
+  const npm = await exports.utilityInstalled("npm")
+  if (!yarn && !npm) {
+    throw new Error("Must have yarn or npm installed to run build.")
+  }
+  const npmCmd = command === "install" ? `npm ${command}` : `npm run ${command}`
+  const cmd = yarn ? `yarn ${command} --ignore-engines` : npmCmd
+  await exports.exec(cmd, dir)
+}

package/src/hosting/genUser.ts

@@ -0,0 +1,23 @@
+const { success } = require("../utils")
+const { updateDockerComposeService } = require("./utils")
+const randomString = require("randomstring")
+const { GENERATED_USER_EMAIL } = require("../constants")
+import { DockerCompose } from "./types"
+
+export async function generateUser(password: string | null, silent: boolean) {
+  const email = GENERATED_USER_EMAIL
+  if (!password) {
+    password = randomString.generate({ length: 6 })
+  }
+  updateDockerComposeService((service: DockerCompose) => {
+    service.environment["BB_ADMIN_USER_EMAIL"] = email
+    service.environment["BB_ADMIN_USER_PASSWORD"] = password as string
+  })
+  if (!silent) {
+    console.log(
+      success(
+        `User admin credentials configured, access with email: ${email} - password: ${password}`
+      )
+    )
+  }
+}

package/src/hosting/index.ts

@@ -0,0 +1,48 @@
+import { Command } from "../structures/Command"
+import { CommandWord } from "../constants"
+import { init } from "./init"
+import { start } from "./start"
+import { stop } from "./stop"
+import { status } from "./status"
+import { update } from "./update"
+import { generateUser } from "./genUser"
+import { watchPlugins } from "./watch"
+
+export default new Command(`${CommandWord.HOSTING}`)
+  .addHelp("Controls self hosting on the Budibase platform.")
+  .addSubOption(
+    "--init [type]",
+    "Configure a self hosted platform in current directory, type can be unspecified, 'quick' or 'single'.",
+    init
+  )
+  .addSubOption(
+    "--start",
+    "Start the configured platform in current directory.",
+    start
+  )
+  .addSubOption(
+    "--status",
+    "Check the status of currently running services.",
+    status
+  )
+  .addSubOption(
+    "--stop",
+    "Stop the configured platform in the current directory.",
+    stop
+  )
+  .addSubOption(
+    "--update",
+    "Update the Budibase images to the latest version.",
+    update
+  )
+  .addSubOption(
+    "--watch-plugin-dir [directory]",
+    "Add plugin directory watching to a Budibase install.",
+    watchPlugins
+  )
+  .addSubOption(
+    "--gen-user",
+    "Create an admin user automatically as part of first start.",
+    generateUser
+  )
+  .addSubOption("--single", "Specify this with init to use the single image.")

package/src/hosting/init.ts

@@ -0,0 +1,76 @@
+import { InitType, AnalyticsEvent } from "../constants"
+import { confirmation } from "../questions"
+import { captureEvent } from "../events"
+import * as makeFiles from "./makeFiles"
+import { parseEnv } from "../utils"
+import { checkDockerConfigured, downloadDockerCompose } from "./utils"
+import { watchPlugins } from "./watch"
+import { generateUser } from "./genUser"
+import fetch from "node-fetch"
+
+const DO_USER_DATA_URL = "http://169.254.169.254/metadata/v1/user-data"
+
+async function getInitConfig(type: string, isQuick: boolean, port: number) {
+  const config: any = isQuick ? makeFiles.QUICK_CONFIG : {}
+  if (type === InitType.DIGITAL_OCEAN) {
+    try {
+      const output = await fetch(DO_USER_DATA_URL)
+      const data = await output.text()
+      const response = parseEnv(data)
+      for (let [key, value] of Object.entries(makeFiles.ConfigMap)) {
+        if (response[key]) {
+          config[value as string] = response[key]
+        }
+      }
+    } catch (err) {
+      // don't need to handle error, just don't do anything
+    }
+  }
+  // override port
+  if (port) {
+    config[makeFiles.ConfigMap.MAIN_PORT] = port
+  }
+  return config
+}
+
+export async function init(opts: any) {
+  let type, isSingle, watchDir, genUser, port, silent
+  if (typeof opts === "string") {
+    type = opts
+  } else {
+    type = opts["init"]
+    isSingle = opts["single"]
+    watchDir = opts["watchPluginDir"]
+    genUser = opts["genUser"]
+    port = opts["port"]
+    silent = opts["silent"]
+  }
+  const isQuick = type === InitType.QUICK || type === InitType.DIGITAL_OCEAN
+  await checkDockerConfigured()
+  if (!isQuick) {
+    const shouldContinue = await confirmation(
+      "This will create multiple files in current directory, should continue?"
+    )
+    if (!shouldContinue) {
+      console.log("Stopping.")
+      return
+    }
+  }
+  captureEvent(AnalyticsEvent.SelfHostInit, {
+    type,
+  })
+  const config = await getInitConfig(type, isQuick, port)
+  if (!isSingle) {
+    await downloadDockerCompose()
+    await makeFiles.makeEnv(config, silent)
+  } else {
+    await makeFiles.makeSingleCompose(config, silent)
+  }
+  if (watchDir) {
+    await watchPlugins(watchDir, silent)
+  }
+  if (genUser) {
+    const inputPassword = typeof genUser === "string" ? genUser : null
+    await generateUser(inputPassword, silent)
+  }
+}