@naturalcycles/backend-lib 9.79.0 → 9.81.0

package/dist/deploy/deployPrepare.js

@@ -1,85 +0,0 @@
-import { loadEnvFileIfExists } from '@naturalcycles/nodejs-lib';
-import { dimGrey } from '@naturalcycles/nodejs-lib/colors';
-import { fs2 } from '@naturalcycles/nodejs-lib/fs2';
-import { kpySync } from '@naturalcycles/nodejs-lib/kpy';
-import { srcDir } from '../paths.cnst.js';
-import { getBackendCfg } from './backend.cfg.util.js';
-import { createAndSaveAppYaml, createAndSaveDeployInfo } from './deploy.util.js';
-export const deployPrepareCliOptions = {
-    projectDir: {
-        type: 'string',
-        default: '.',
-    },
-    targetDir: {
-        type: 'string',
-        default: `./tmp/deploy`,
-    },
-    createNpmrc: {
-        type: 'boolean',
-        desc: 'Create .npmrc in targetDir if NPM_TOKEN env var is set',
-        default: true,
-    },
-    appYamlPassEnv: {
-        type: 'string',
-        desc: 'Comma-separated list of env variables that will be passed to app.yaml from process.env',
-    },
-};
-const DEFAULT_FILES = [
-    'dist',
-    // '!dist/test',
-    // '!dist/**/*.test.*',
-    // '!dist/**/*.mock.*',
-    '!dist/**/*.script.*',
-    'src', // For Sentry
-    '!src/test',
-    '!src/**/*.test.*',
-    '!src/**/*.mock.*',
-    '!src/**/*.script.*',
-    '!**/__snapshots__',
-    '!**/__exclude',
-    'static',
-    'secret/**/*.enc', // encrypted secrets
-    'package.json',
-    'yarn.lock',
-    '.yarnrc',
-    'pnpm-lock.yaml',
-    'pnpm-workspace.yaml',
-    // 'tsconfig.json', // for path-mapping to work!
-    // 'tsconfig.dist.json',
-    '.gcloudignore',
-    'app.yaml',
-    'patches', // to allow patch-package
-    'resources',
-];
-const defaultFilesDir = `${srcDir}/deploy/files-default`;
-export async function deployPrepare(opt = {}) {
-    loadEnvFileIfExists();
-    const { projectDir = '.', targetDir = './tmp/deploy', createNpmrc = true } = opt;
-    const backendCfg = getBackendCfg(projectDir);
-    const inputPatterns = backendCfg.files || DEFAULT_FILES;
-    const appYamlPassEnv = opt.appYamlPassEnv || backendCfg.appYamlPassEnv;
-    console.log(`1. Copy files to ${dimGrey(targetDir)}`);
-    // Clean targetDir
-    fs2.emptyDir(targetDir);
-    kpySync({
-        baseDir: defaultFilesDir,
-        outputDir: targetDir,
-        dotfiles: true,
-    });
-    kpySync({
-        baseDir: projectDir,
-        inputPatterns,
-        outputDir: targetDir,
-        dotfiles: true,
-    });
-    const { NPM_TOKEN } = process.env;
-    if (NPM_TOKEN && createNpmrc) {
-        const npmrcPath = `${targetDir}/.npmrc`;
-        const npmrc = `//registry.npmjs.org/:_authToken=${NPM_TOKEN}`;
-        fs2.writeFile(npmrcPath, npmrc);
-    }
-    console.log(`2. Generate ${dimGrey('deployInfo.json')} and ${dimGrey('app.yaml')} in targetDir`);
-    const deployInfo = await createAndSaveDeployInfo(backendCfg, targetDir);
-    createAndSaveAppYaml(backendCfg, deployInfo, projectDir, targetDir, appYamlPassEnv);
-    return deployInfo;
-}

package/src/bin/deploy-gae.ts

@@ -1,14 +0,0 @@
-import { _parseArgs } from '@naturalcycles/nodejs-lib/args'
-import { runScript } from '@naturalcycles/nodejs-lib/runScript'
-import { deployGae } from '../deploy/deployGae.js'
-import { deployHealthCheckCliOptions } from '../deploy/deployHealthCheck.js'
-import { deployPrepareCliOptions } from '../deploy/deployPrepare.js'
-
-runScript(async () => {
-  const opt = _parseArgs({
-    ...deployPrepareCliOptions,
-    ...deployHealthCheckCliOptions,
-  })
-
-  await deployGae(opt)
-})

package/src/bin/deploy-prepare.ts

@@ -1,24 +0,0 @@
-/*
-
-yarn deploy-prepare
-
- */
-
-import { _parseArgs } from '@naturalcycles/nodejs-lib/args'
-import { runScript } from '@naturalcycles/nodejs-lib/runScript'
-import { deployPrepare, deployPrepareCliOptions } from '../deploy/deployPrepare.js'
-
-runScript(async () => {
-  const opt = _parseArgs(deployPrepareCliOptions)
-
-  await deployPrepare(opt)
-})
-
-// deploy strategy
-// gae project: from config
-// gae service: from config -- branch name
-// gae version: automatic form date
-
-// yarn build-prod
-// yarn deploy-prepare && ./tmp/deploy/app.yaml && ./tmp/deploy/deployInfo.json && json2env
-//

package/src/bin/undeploy-gae.ts

@@ -1,15 +0,0 @@
-import { _parseArgs } from '@naturalcycles/nodejs-lib/args'
-import { runScript } from '@naturalcycles/nodejs-lib/runScript'
-import { undeployGae } from '../deploy/deployGae.js'
-
-runScript(async () => {
-  const { branch } = _parseArgs({
-    branch: {
-      type: 'string',
-      demandOption: true,
-      desc: `Because Github Actions delete event happens after the branch is already deleted - you need to pass it manually`,
-    },
-  })
-
-  await undeployGae(branch)
-})

package/src/deploy/backend.cfg.util.ts

@@ -1,63 +0,0 @@
-import type { StringMap } from '@naturalcycles/js-lib/types'
-import { j } from '@naturalcycles/nodejs-lib/ajv'
-import { fs2 } from '@naturalcycles/nodejs-lib/fs2'
-import { yaml2 } from '@naturalcycles/nodejs-lib/yaml2'
-
-export interface BackendCfg {
-  gaeProject: string
-  gaeProjectByBranch?: StringMap
-
-  /**
-   * @example default
-   */
-  gaeService: string
-  gaeServiceByBranch?: StringMap
-
-  /**
-   * List of file patterns to include in deployment.
-   */
-  files?: string[]
-
-  appEnvDefault: string
-  appEnvByBranch?: StringMap
-
-  /**
-   * List of branches to use timestamps in gae version names (to keep previous versions).
-   */
-  branchesWithTimestampVersions?: string[]
-
-  /**
-   * If true - branch names are not passed into deployed urls as is, but are hashed.
-   */
-  hashedBranches?: boolean
-
-  /**
-   * Comma-separated list of env variables that will be passed to app.yaml from process.env
-   */
-  appYamlPassEnv?: string
-}
-
-export const backendCfgSchema = j.object<BackendCfg>({
-  gaeProject: j.string(),
-  gaeProjectByBranch: j.object.stringMap(j.string()).optional(),
-  gaeService: j.string(),
-  gaeServiceByBranch: j.object.stringMap(j.string()).optional(),
-  files: j.array(j.string()).optional(),
-  appEnvDefault: j.string(),
-  appEnvByBranch: j.object.stringMap(j.string()).optional(),
-  branchesWithTimestampVersions: j.array(j.string()).optional(),
-  hashedBranches: j.boolean().optional(),
-  appYamlPassEnv: j.string().optional(),
-})
-
-export function getBackendCfg(projectDir = '.'): BackendCfg {
-  const backendCfgYamlPath = `${projectDir}/backend.cfg.yaml`
-
-  fs2.requireFileToExist(backendCfgYamlPath)
-
-  const backendCfg: BackendCfg = {
-    ...yaml2.readYaml(backendCfgYamlPath),
-  }
-
-  return backendCfgSchema.validate(backendCfg)
-}

package/src/deploy/deploy.model.ts

@@ -1,32 +0,0 @@
-import type { UnixTimestamp } from '@naturalcycles/js-lib/types'
-
-export interface DeployInfo {
-  gaeProject: string
-  gaeService: string
-  gaeVersion: string
-  versionUrl: string
-  serviceUrl: string
-  gitRev: string
-  gitBranch: string
-
-  /**
-   * Unix timestamp of deployInfo.json being generated.
-   */
-  ts: UnixTimestamp
-}
-
-export interface AppYaml extends Record<string, any> {
-  runtime: string
-  service: string
-  inbound_services?: string[]
-  instance_class?: string
-  automatic_scaling?: {
-    min_instances?: number
-    max_instances?: number
-  }
-  env_variables: {
-    APP_ENV: string
-    DEBUG: string
-    [k: string]: string
-  }
-}

package/src/deploy/deploy.util.ts

@@ -1,214 +0,0 @@
-import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js'
-import { _assert } from '@naturalcycles/js-lib/error/assert.js'
-import { _mapValues, _merge } from '@naturalcycles/js-lib/object/object.util.js'
-import { _truncate } from '@naturalcycles/js-lib/string/string.util.js'
-import { sha256 } from '@naturalcycles/nodejs-lib'
-import { dimGrey, white } from '@naturalcycles/nodejs-lib/colors'
-import { fs2 } from '@naturalcycles/nodejs-lib/fs2'
-import { yaml2 } from '@naturalcycles/nodejs-lib/yaml2'
-import type { BackendCfg } from './backend.cfg.util.js'
-import type { AppYaml, DeployInfo } from './deploy.model.js'
-
-const APP_YAML_DEFAULT = (): AppYaml => ({
-  runtime: 'nodejs22',
-  service: 'default',
-  inbound_services: ['warmup'],
-  instance_class: 'F1',
-  automatic_scaling: {
-    min_instances: 0,
-    max_instances: 1,
-  },
-  env_variables: {
-    APP_ENV: 'prod',
-    DEBUG: 'app*,nc:*',
-    DEBUG_HIDE_DATE: 'true',
-    // DEBUG_COLORS: 'true',
-    TZ: 'UTC',
-  },
-})
-
-export async function createAndSaveDeployInfo(
-  backendCfg: BackendCfg,
-  targetDir: string,
-): Promise<DeployInfo> {
-  const deployInfo = await createDeployInfo(backendCfg)
-  const deployInfoPath = `${targetDir}/deployInfo.json`
-  fs2.writeJson(deployInfoPath, deployInfo, { spaces: 2 })
-  console.log(`saved ${dimGrey(deployInfoPath)}`)
-  return deployInfo
-}
-
-export async function createDeployInfo(
-  backendCfg: BackendCfg,
-  overrideBranch?: string,
-): Promise<DeployInfo> {
-  const { simpleGit } = await import('simple-git') // lazy load
-  const git = simpleGit('.')
-
-  const now = localTime.now()
-  const gitBranch = overrideBranch || (await git.status()).current!
-  const gitRev = (await git.revparse(['HEAD'])).slice(0, 7)
-
-  let {
-    gaeProject,
-    gaeProjectByBranch = {},
-    gaeService,
-    gaeServiceByBranch = {},
-    branchesWithTimestampVersions = [],
-  } = backendCfg
-
-  gaeProject = gaeProjectByBranch[gitBranch] || gaeProject
-
-  if (gaeServiceByBranch[gitBranch]) {
-    gaeService = validateGAEServiceName(gaeServiceByBranch[gitBranch])
-  } else {
-    let branchName = gitBranch
-
-    if (backendCfg.hashedBranches) {
-      // Obfuscates the branch name by hashing it.
-      // If there are Jira issue names in the branch name, the first one found will be used as a prefix.
-      const jiraIssue = gitBranch.match(/([Dd][Ee][Vv]-\d+)/)?.[0]
-      const branchHash = sha256(gitBranch).slice(0, 10)
-      branchName = [jiraIssue, branchHash].filter(Boolean).join('-')
-    }
-
-    gaeService = validateGAEServiceName([branchName, gaeService].join('--'))
-  }
-
-  let gaeVersion = '1'
-
-  if (branchesWithTimestampVersions.includes(gitBranch)) {
-    // May only contain lowercase letters, digits, and hyphens. Must begin and end with a letter or digit. Must not exceed 63 characters.
-    gaeVersion = [
-      // now.format('YYYYMMDD-HHmm'), // 20190521-1721
-      now.toStringCompact().replace('_', '-'),
-      gitRev,
-    ].join('-')
-  }
-
-  const versionUrl = `https://${[gaeVersion, gaeService, gaeProject].join('-dot-')}.appspot.com`
-
-  // Check the 63-char limit
-  const versionUrlString = [gaeVersion, gaeService, gaeProject].join('-dot-')
-  _assert(
-    versionUrlString.length <= 63,
-    `versionUrl length should be <= 63 characters, but it's ${versionUrlString.length} instead: ${versionUrlString}`,
-  )
-
-  const serviceUrl = `https://${[gaeService, gaeProject].join('-dot-')}.appspot.com`
-
-  const deployInfo: DeployInfo = {
-    gaeProject,
-    gaeService,
-    gaeVersion,
-    versionUrl,
-    serviceUrl,
-    gitBranch,
-    gitRev,
-    ts: now.unix,
-  }
-
-  console.log({ deployInfo })
-
-  return deployInfo
-}
-
-export function createAndSaveAppYaml(
-  backendCfg: BackendCfg,
-  deployInfo: DeployInfo,
-  projectDir: string,
-  targetDir: string,
-  appYamlPassEnv = '',
-): AppYaml {
-  const appYaml = createAppYaml(backendCfg, deployInfo, projectDir, appYamlPassEnv)
-  const appYamlPath = `${targetDir}/app.yaml`
-  yaml2.writeYaml(appYamlPath, appYaml)
-  console.log(`saved ${dimGrey(appYamlPath)}`)
-  return appYaml
-}
-
-export function createAppYaml(
-  backendCfg: BackendCfg,
-  deployInfo: DeployInfo,
-  projectDir: string,
-  appYamlPassEnv = '',
-): AppYaml {
-  const { appEnvDefault, appEnvByBranch = {} } = backendCfg
-  const { gaeService: service, gitBranch } = deployInfo
-
-  const { APP_ENV: processAppEnv } = process.env
-  const APP_ENV = processAppEnv || appEnvByBranch[gitBranch] || appEnvDefault
-  if (processAppEnv) {
-    console.log(`using APP_ENV=${dimGrey(processAppEnv)} from process.env`)
-  }
-
-  const appYaml = APP_YAML_DEFAULT()
-
-  // Check existing app.yaml
-  const appYamlPath = `${projectDir}/app.yaml`
-  if (fs2.pathExists(appYamlPath)) {
-    console.log(`merging-in ${dimGrey(appYamlPath)}`)
-    _merge(appYaml, yaml2.readYaml(appYamlPath))
-  }
-
-  const appEnvYamlPath = `${projectDir}/app.${APP_ENV}.yaml`
-  if (fs2.pathExists(appEnvYamlPath)) {
-    console.log(`merging-in ${dimGrey(appEnvYamlPath)}`)
-    _merge(appYaml, yaml2.readYaml(appEnvYamlPath))
-  }
-
-  // appYamlPassEnv
-  const passEnv = appYamlPassEnv
-    .split(',')
-    .filter(Boolean)
-    // oxlint-disable-next-line unicorn/no-array-reduce
-    .reduce<Record<string, string>>((map, key) => {
-      const v = process.env[key]
-      if (!v) {
-        throw new Error(
-          `appYamlPassEnv.${key} is requested, but process.env.${key} is not defined!`,
-        )
-      }
-      map[key] = v
-      return map
-    }, {})
-
-  if (Object.keys(passEnv).length) {
-    console.log(
-      `will merge ${white(
-        String(Object.keys(passEnv).length),
-      )} process.env keys to app.yaml: ${dimGrey(Object.keys(passEnv).join(', '))}`,
-    )
-  }
-
-  _merge(appYaml, {
-    service,
-    env_variables: {
-      APP_ENV,
-      ...passEnv,
-    },
-  })
-
-  // Redacted appYaml to not show up secrets
-  console.log({
-    appYaml: redactedAppYaml(appYaml),
-  })
-
-  return appYaml
-}
-
-function redactedAppYaml(appYaml: AppYaml): AppYaml {
-  return {
-    ...appYaml,
-    env_variables: _mapValues(appYaml.env_variables || {}, (_k, v) => _truncate(v, 7)),
-  }
-}
-
-export function validateGAEServiceName(serviceName: string): string {
-  // May only contain lowercase letters, digits, and hyphens. Must begin and end with a letter or digit. Must not exceed 63 characters.
-  return serviceName
-    .replaceAll('_', '-')
-    .toLowerCase()
-    .replaceAll(/[^0-9a-z-]/gi, '')
-    .slice(0, 40)
-}

package/src/deploy/deployGae.ts

@@ -1,127 +0,0 @@
-import { _anyToError } from '@naturalcycles/js-lib/error'
-import { pRetry } from '@naturalcycles/js-lib/promise/pRetry.js'
-import { _objectAssign } from '@naturalcycles/js-lib/types'
-import { appendToGithubSummary } from '@naturalcycles/nodejs-lib/env'
-import { exec2 } from '@naturalcycles/nodejs-lib/exec2'
-import { fs2 } from '@naturalcycles/nodejs-lib/fs2'
-import { getBackendCfg } from './backend.cfg.util.js'
-import { createDeployInfo } from './deploy.util.js'
-import type { DeployHealthCheckOptions } from './deployHealthCheck.js'
-import { deployHealthCheck } from './deployHealthCheck.js'
-import type { DeployPrepareOptions } from './deployPrepare.js'
-import { deployPrepare } from './deployPrepare.js'
-
-export interface DeployGaeOptions extends DeployPrepareOptions, DeployHealthCheckOptions {}
-
-export async function deployGae(opt: DeployGaeOptions = {}): Promise<void> {
-  const { logOnFailure, logOnSuccess } = opt
-
-  // 1. build
-
-  const packageManager = detectPackageManager()
-
-  exec2.spawn(`${packageManager} build`)
-
-  // 2. deploy-prepare
-
-  const deployInfo = await deployPrepare()
-
-  const targetDir = './tmp/deploy'
-  const appYamlPath = `${targetDir}/app.yaml`
-
-  const { gaeProject, gaeService, gaeVersion, versionUrl, serviceUrl } = deployInfo
-  _objectAssign(opt, {
-    gaeProject,
-    gaeService,
-    gaeVersion,
-  })
-
-  await pRetry(
-    async () => {
-      try {
-        exec2.spawn(
-          `gcloud app deploy ${appYamlPath} --project ${gaeProject} --version ${gaeVersion} --quiet --no-promote`,
-        )
-      } catch (err) {
-        if (logOnFailure) {
-          logs(gaeProject, gaeService, gaeVersion)
-        }
-        throw err
-      }
-    },
-    {
-      name: 'deploy',
-      maxAttempts: 2,
-      delay: 30_000,
-      // todo: this doesn't work, as the error is different from what is logged.
-      // We shoud somehow capture the logged text
-      predicate: err => _anyToError(err).message.includes('operation is already in progress'),
-    },
-  )
-
-  // Health check (versionUrl)
-  // yarn deploy-health-check --url $deployInfo_versionUrl --repeat 3 --timeoutSec 180 --intervalSec 2
-  await deployHealthCheck(versionUrl, opt)
-
-  // Only if "timestamped version" is used ('1' is default)
-  if (gaeVersion !== '1') {
-    // Rollout (promote versionUrl to serviceUrl)
-    // gcloud app services set-traffic $deployInfo_gaeService --project $deployInfo_gaeProject --splits $deployInfo_gaeVersion=1 --quiet
-    exec2.spawn(
-      `gcloud app services set-traffic ${gaeService} --project ${gaeProject} --splits ${gaeVersion}=1 --quiet`,
-    )
-
-    // Health check (serviceUrl)
-    // yarn deploy-health-check --url $deployInfo_serviceUrl --repeat 3 --timeoutSec 60 --intervalSec 2
-    await deployHealthCheck(serviceUrl, opt)
-  }
-
-  appendToGithubSummary(`Deployed ${serviceUrl}`)
-
-  // Logs
-  if (logOnSuccess) {
-    logs(gaeProject, gaeService, gaeVersion)
-  }
-}
-
-/**
- * Undeploys/removes the GAE service/version, using the same rules as deployGae.
- * Detects the service/version by the same criteria: branch name, backend.cfg.yaml, etc.
- */
-export async function undeployGae(branch: string): Promise<void> {
-  const projectDir = '.'
-  const backendCfg = getBackendCfg(projectDir)
-
-  const { gaeProject, gaeService, gaeVersion } = await createDeployInfo(backendCfg, branch)
-
-  // if (prod) {
-  //   console.log(`undeployGae (branch: ${branch}): not removing prod version (safety check)`)
-  //   return
-  // }
-
-  console.log(
-    `undeployGae (branch: ${branch}): going to remove ${gaeProject}/${gaeService}/${gaeVersion}`,
-  )
-
-  exec2.spawn(
-    `gcloud app versions delete --project ${gaeProject} --service ${gaeService} ${gaeVersion} --quiet`,
-  )
-
-  appendToGithubSummary(`removed ${gaeProject}/${gaeService}/${gaeVersion}`)
-}
-
-function logs(gaeProject: string, gaeService: string, gaeVersion: string): void {
-  try {
-    exec2.spawn(
-      `gcloud app logs read --project ${gaeProject} --service ${gaeService} --version ${gaeVersion}`,
-    )
-  } catch {}
-}
-
-function detectPackageManager(): string {
-  const pj = fs2.readText('package.json')
-  if (pj.includes('pnpm')) {
-    return 'pnpm'
-  }
-  return 'yarn' // yarn is still the default
-}

package/src/deploy/deployInfo.util.ts

@@ -1,27 +0,0 @@
-import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js'
-import { _memoized } from '@naturalcycles/js-lib/decorators'
-import { fs2 } from '@naturalcycles/nodejs-lib/fs2'
-import type { DeployInfo } from './deploy.model.js'
-
-export const getDeployInfo = _memoized((projectDir: string): DeployInfo => {
-  const deployInfoPath = `${projectDir}/deployInfo.json`
-  try {
-    return fs2.readJson(deployInfoPath)
-  } catch {
-    // console.error(`cannot read ${deployInfoPath}, returning empty version`)
-    return getDeployInfoStub()
-  }
-})
-
-function getDeployInfoStub(stub = ''): DeployInfo {
-  return {
-    gaeProject: stub,
-    gaeService: stub,
-    gaeVersion: stub,
-    serviceUrl: stub,
-    versionUrl: stub,
-    gitBranch: stub,
-    gitRev: stub,
-    ts: localTime.nowUnix(),
-  }
-}