@naturalcycles/backend-lib 9.79.0 → 9.80.0

package/src/deploy/deploy.util.ts

@@ -1,214 +0,0 @@
-import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js'
-import { _assert } from '@naturalcycles/js-lib/error/assert.js'
-import { _mapValues, _merge } from '@naturalcycles/js-lib/object/object.util.js'
-import { _truncate } from '@naturalcycles/js-lib/string/string.util.js'
-import { sha256 } from '@naturalcycles/nodejs-lib'
-import { dimGrey, white } from '@naturalcycles/nodejs-lib/colors'
-import { fs2 } from '@naturalcycles/nodejs-lib/fs2'
-import { yaml2 } from '@naturalcycles/nodejs-lib/yaml2'
-import type { BackendCfg } from './backend.cfg.util.js'
-import type { AppYaml, DeployInfo } from './deploy.model.js'
-
-const APP_YAML_DEFAULT = (): AppYaml => ({
-  runtime: 'nodejs22',
-  service: 'default',
-  inbound_services: ['warmup'],
-  instance_class: 'F1',
-  automatic_scaling: {
-    min_instances: 0,
-    max_instances: 1,
-  },
-  env_variables: {
-    APP_ENV: 'prod',
-    DEBUG: 'app*,nc:*',
-    DEBUG_HIDE_DATE: 'true',
-    // DEBUG_COLORS: 'true',
-    TZ: 'UTC',
-  },
-})
-
-export async function createAndSaveDeployInfo(
-  backendCfg: BackendCfg,
-  targetDir: string,
-): Promise<DeployInfo> {
-  const deployInfo = await createDeployInfo(backendCfg)
-  const deployInfoPath = `${targetDir}/deployInfo.json`
-  fs2.writeJson(deployInfoPath, deployInfo, { spaces: 2 })
-  console.log(`saved ${dimGrey(deployInfoPath)}`)
-  return deployInfo
-}
-
-export async function createDeployInfo(
-  backendCfg: BackendCfg,
-  overrideBranch?: string,
-): Promise<DeployInfo> {
-  const { simpleGit } = await import('simple-git') // lazy load
-  const git = simpleGit('.')
-
-  const now = localTime.now()
-  const gitBranch = overrideBranch || (await git.status()).current!
-  const gitRev = (await git.revparse(['HEAD'])).slice(0, 7)
-
-  let {
-    gaeProject,
-    gaeProjectByBranch = {},
-    gaeService,
-    gaeServiceByBranch = {},
-    branchesWithTimestampVersions = [],
-  } = backendCfg
-
-  gaeProject = gaeProjectByBranch[gitBranch] || gaeProject
-
-  if (gaeServiceByBranch[gitBranch]) {
-    gaeService = validateGAEServiceName(gaeServiceByBranch[gitBranch])
-  } else {
-    let branchName = gitBranch
-
-    if (backendCfg.hashedBranches) {
-      // Obfuscates the branch name by hashing it.
-      // If there are Jira issue names in the branch name, the first one found will be used as a prefix.
-      const jiraIssue = gitBranch.match(/([Dd][Ee][Vv]-\d+)/)?.[0]
-      const branchHash = sha256(gitBranch).slice(0, 10)
-      branchName = [jiraIssue, branchHash].filter(Boolean).join('-')
-    }
-
-    gaeService = validateGAEServiceName([branchName, gaeService].join('--'))
-  }
-
-  let gaeVersion = '1'
-
-  if (branchesWithTimestampVersions.includes(gitBranch)) {
-    // May only contain lowercase letters, digits, and hyphens. Must begin and end with a letter or digit. Must not exceed 63 characters.
-    gaeVersion = [
-      // now.format('YYYYMMDD-HHmm'), // 20190521-1721
-      now.toStringCompact().replace('_', '-'),
-      gitRev,
-    ].join('-')
-  }
-
-  const versionUrl = `https://${[gaeVersion, gaeService, gaeProject].join('-dot-')}.appspot.com`
-
-  // Check the 63-char limit
-  const versionUrlString = [gaeVersion, gaeService, gaeProject].join('-dot-')
-  _assert(
-    versionUrlString.length <= 63,
-    `versionUrl length should be <= 63 characters, but it's ${versionUrlString.length} instead: ${versionUrlString}`,
-  )
-
-  const serviceUrl = `https://${[gaeService, gaeProject].join('-dot-')}.appspot.com`
-
-  const deployInfo: DeployInfo = {
-    gaeProject,
-    gaeService,
-    gaeVersion,
-    versionUrl,
-    serviceUrl,
-    gitBranch,
-    gitRev,
-    ts: now.unix,
-  }
-
-  console.log({ deployInfo })
-
-  return deployInfo
-}
-
-export function createAndSaveAppYaml(
-  backendCfg: BackendCfg,
-  deployInfo: DeployInfo,
-  projectDir: string,
-  targetDir: string,
-  appYamlPassEnv = '',
-): AppYaml {
-  const appYaml = createAppYaml(backendCfg, deployInfo, projectDir, appYamlPassEnv)
-  const appYamlPath = `${targetDir}/app.yaml`
-  yaml2.writeYaml(appYamlPath, appYaml)
-  console.log(`saved ${dimGrey(appYamlPath)}`)
-  return appYaml
-}
-
-export function createAppYaml(
-  backendCfg: BackendCfg,
-  deployInfo: DeployInfo,
-  projectDir: string,
-  appYamlPassEnv = '',
-): AppYaml {
-  const { appEnvDefault, appEnvByBranch = {} } = backendCfg
-  const { gaeService: service, gitBranch } = deployInfo
-
-  const { APP_ENV: processAppEnv } = process.env
-  const APP_ENV = processAppEnv || appEnvByBranch[gitBranch] || appEnvDefault
-  if (processAppEnv) {
-    console.log(`using APP_ENV=${dimGrey(processAppEnv)} from process.env`)
-  }
-
-  const appYaml = APP_YAML_DEFAULT()
-
-  // Check existing app.yaml
-  const appYamlPath = `${projectDir}/app.yaml`
-  if (fs2.pathExists(appYamlPath)) {
-    console.log(`merging-in ${dimGrey(appYamlPath)}`)
-    _merge(appYaml, yaml2.readYaml(appYamlPath))
-  }
-
-  const appEnvYamlPath = `${projectDir}/app.${APP_ENV}.yaml`
-  if (fs2.pathExists(appEnvYamlPath)) {
-    console.log(`merging-in ${dimGrey(appEnvYamlPath)}`)
-    _merge(appYaml, yaml2.readYaml(appEnvYamlPath))
-  }
-
-  // appYamlPassEnv
-  const passEnv = appYamlPassEnv
-    .split(',')
-    .filter(Boolean)
-    // oxlint-disable-next-line unicorn/no-array-reduce
-    .reduce<Record<string, string>>((map, key) => {
-      const v = process.env[key]
-      if (!v) {
-        throw new Error(
-          `appYamlPassEnv.${key} is requested, but process.env.${key} is not defined!`,
-        )
-      }
-      map[key] = v
-      return map
-    }, {})
-
-  if (Object.keys(passEnv).length) {
-    console.log(
-      `will merge ${white(
-        String(Object.keys(passEnv).length),
-      )} process.env keys to app.yaml: ${dimGrey(Object.keys(passEnv).join(', '))}`,
-    )
-  }
-
-  _merge(appYaml, {
-    service,
-    env_variables: {
-      APP_ENV,
-      ...passEnv,
-    },
-  })
-
-  // Redacted appYaml to not show up secrets
-  console.log({
-    appYaml: redactedAppYaml(appYaml),
-  })
-
-  return appYaml
-}
-
-function redactedAppYaml(appYaml: AppYaml): AppYaml {
-  return {
-    ...appYaml,
-    env_variables: _mapValues(appYaml.env_variables || {}, (_k, v) => _truncate(v, 7)),
-  }
-}
-
-export function validateGAEServiceName(serviceName: string): string {
-  // May only contain lowercase letters, digits, and hyphens. Must begin and end with a letter or digit. Must not exceed 63 characters.
-  return serviceName
-    .replaceAll('_', '-')
-    .toLowerCase()
-    .replaceAll(/[^0-9a-z-]/gi, '')
-    .slice(0, 40)
-}

package/src/deploy/deployGae.ts

@@ -1,127 +0,0 @@
-import { _anyToError } from '@naturalcycles/js-lib/error'
-import { pRetry } from '@naturalcycles/js-lib/promise/pRetry.js'
-import { _objectAssign } from '@naturalcycles/js-lib/types'
-import { appendToGithubSummary } from '@naturalcycles/nodejs-lib/env'
-import { exec2 } from '@naturalcycles/nodejs-lib/exec2'
-import { fs2 } from '@naturalcycles/nodejs-lib/fs2'
-import { getBackendCfg } from './backend.cfg.util.js'
-import { createDeployInfo } from './deploy.util.js'
-import type { DeployHealthCheckOptions } from './deployHealthCheck.js'
-import { deployHealthCheck } from './deployHealthCheck.js'
-import type { DeployPrepareOptions } from './deployPrepare.js'
-import { deployPrepare } from './deployPrepare.js'
-
-export interface DeployGaeOptions extends DeployPrepareOptions, DeployHealthCheckOptions {}
-
-export async function deployGae(opt: DeployGaeOptions = {}): Promise<void> {
-  const { logOnFailure, logOnSuccess } = opt
-
-  // 1. build
-
-  const packageManager = detectPackageManager()
-
-  exec2.spawn(`${packageManager} build`)
-
-  // 2. deploy-prepare
-
-  const deployInfo = await deployPrepare()
-
-  const targetDir = './tmp/deploy'
-  const appYamlPath = `${targetDir}/app.yaml`
-
-  const { gaeProject, gaeService, gaeVersion, versionUrl, serviceUrl } = deployInfo
-  _objectAssign(opt, {
-    gaeProject,
-    gaeService,
-    gaeVersion,
-  })
-
-  await pRetry(
-    async () => {
-      try {
-        exec2.spawn(
-          `gcloud app deploy ${appYamlPath} --project ${gaeProject} --version ${gaeVersion} --quiet --no-promote`,
-        )
-      } catch (err) {
-        if (logOnFailure) {
-          logs(gaeProject, gaeService, gaeVersion)
-        }
-        throw err
-      }
-    },
-    {
-      name: 'deploy',
-      maxAttempts: 2,
-      delay: 30_000,
-      // todo: this doesn't work, as the error is different from what is logged.
-      // We shoud somehow capture the logged text
-      predicate: err => _anyToError(err).message.includes('operation is already in progress'),
-    },
-  )
-
-  // Health check (versionUrl)
-  // yarn deploy-health-check --url $deployInfo_versionUrl --repeat 3 --timeoutSec 180 --intervalSec 2
-  await deployHealthCheck(versionUrl, opt)
-
-  // Only if "timestamped version" is used ('1' is default)
-  if (gaeVersion !== '1') {
-    // Rollout (promote versionUrl to serviceUrl)
-    // gcloud app services set-traffic $deployInfo_gaeService --project $deployInfo_gaeProject --splits $deployInfo_gaeVersion=1 --quiet
-    exec2.spawn(
-      `gcloud app services set-traffic ${gaeService} --project ${gaeProject} --splits ${gaeVersion}=1 --quiet`,
-    )
-
-    // Health check (serviceUrl)
-    // yarn deploy-health-check --url $deployInfo_serviceUrl --repeat 3 --timeoutSec 60 --intervalSec 2
-    await deployHealthCheck(serviceUrl, opt)
-  }
-
-  appendToGithubSummary(`Deployed ${serviceUrl}`)
-
-  // Logs
-  if (logOnSuccess) {
-    logs(gaeProject, gaeService, gaeVersion)
-  }
-}
-
-/**
- * Undeploys/removes the GAE service/version, using the same rules as deployGae.
- * Detects the service/version by the same criteria: branch name, backend.cfg.yaml, etc.
- */
-export async function undeployGae(branch: string): Promise<void> {
-  const projectDir = '.'
-  const backendCfg = getBackendCfg(projectDir)
-
-  const { gaeProject, gaeService, gaeVersion } = await createDeployInfo(backendCfg, branch)
-
-  // if (prod) {
-  //   console.log(`undeployGae (branch: ${branch}): not removing prod version (safety check)`)
-  //   return
-  // }
-
-  console.log(
-    `undeployGae (branch: ${branch}): going to remove ${gaeProject}/${gaeService}/${gaeVersion}`,
-  )
-
-  exec2.spawn(
-    `gcloud app versions delete --project ${gaeProject} --service ${gaeService} ${gaeVersion} --quiet`,
-  )
-
-  appendToGithubSummary(`removed ${gaeProject}/${gaeService}/${gaeVersion}`)
-}
-
-function logs(gaeProject: string, gaeService: string, gaeVersion: string): void {
-  try {
-    exec2.spawn(
-      `gcloud app logs read --project ${gaeProject} --service ${gaeService} --version ${gaeVersion}`,
-    )
-  } catch {}
-}
-
-function detectPackageManager(): string {
-  const pj = fs2.readText('package.json')
-  if (pj.includes('pnpm')) {
-    return 'pnpm'
-  }
-  return 'yarn' // yarn is still the default
-}

package/src/deploy/deployInfo.util.ts

@@ -1,27 +0,0 @@
-import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js'
-import { _memoized } from '@naturalcycles/js-lib/decorators'
-import { fs2 } from '@naturalcycles/nodejs-lib/fs2'
-import type { DeployInfo } from './deploy.model.js'
-
-export const getDeployInfo = _memoized((projectDir: string): DeployInfo => {
-  const deployInfoPath = `${projectDir}/deployInfo.json`
-  try {
-    return fs2.readJson(deployInfoPath)
-  } catch {
-    // console.error(`cannot read ${deployInfoPath}, returning empty version`)
-    return getDeployInfoStub()
-  }
-})
-
-function getDeployInfoStub(stub = ''): DeployInfo {
-  return {
-    gaeProject: stub,
-    gaeService: stub,
-    gaeVersion: stub,
-    serviceUrl: stub,
-    versionUrl: stub,
-    gitBranch: stub,
-    gitRev: stub,
-    ts: localTime.nowUnix(),
-  }
-}

package/src/deploy/deployPrepare.ts

@@ -1,113 +0,0 @@
-import { loadEnvFileIfExists } from '@naturalcycles/nodejs-lib'
-import type { CliOptions } from '@naturalcycles/nodejs-lib/args'
-import { dimGrey } from '@naturalcycles/nodejs-lib/colors'
-import { fs2 } from '@naturalcycles/nodejs-lib/fs2'
-import { kpySync } from '@naturalcycles/nodejs-lib/kpy'
-import { srcDir } from '../paths.cnst.js'
-import { getBackendCfg } from './backend.cfg.util.js'
-import type { DeployInfo } from './deploy.model.js'
-import { createAndSaveAppYaml, createAndSaveDeployInfo } from './deploy.util.js'
-
-export interface DeployPrepareOptions {
-  projectDir?: string
-  targetDir?: string
-  createNpmrc?: boolean
-
-  /**
-   * Comma-separated list of env variables that will be passed to app.yaml from process.env.
-   * Use it to pass secrets.
-   */
-  appYamlPassEnv?: string
-}
-
-export const deployPrepareCliOptions = {
-  projectDir: {
-    type: 'string',
-    default: '.',
-  },
-  targetDir: {
-    type: 'string',
-    default: `./tmp/deploy`,
-  },
-  createNpmrc: {
-    type: 'boolean',
-    desc: 'Create .npmrc in targetDir if NPM_TOKEN env var is set',
-    default: true,
-  },
-  appYamlPassEnv: {
-    type: 'string',
-    desc: 'Comma-separated list of env variables that will be passed to app.yaml from process.env',
-  },
-} as const satisfies CliOptions
-
-const DEFAULT_FILES = [
-  'dist',
-  // '!dist/test',
-  // '!dist/**/*.test.*',
-  // '!dist/**/*.mock.*',
-  '!dist/**/*.script.*',
-  'src', // For Sentry
-  '!src/test',
-  '!src/**/*.test.*',
-  '!src/**/*.mock.*',
-  '!src/**/*.script.*',
-  '!**/__snapshots__',
-  '!**/__exclude',
-  'static',
-  'secret/**/*.enc', // encrypted secrets
-  'package.json',
-  'yarn.lock',
-  '.yarnrc',
-  'pnpm-lock.yaml',
-  'pnpm-workspace.yaml',
-  // 'tsconfig.json', // for path-mapping to work!
-  // 'tsconfig.dist.json',
-  '.gcloudignore',
-  'app.yaml',
-  'patches', // to allow patch-package
-  'resources',
-]
-
-const defaultFilesDir = `${srcDir}/deploy/files-default`
-
-export async function deployPrepare(opt: DeployPrepareOptions = {}): Promise<DeployInfo> {
-  loadEnvFileIfExists()
-
-  const { projectDir = '.', targetDir = './tmp/deploy', createNpmrc = true } = opt
-
-  const backendCfg = getBackendCfg(projectDir)
-  const inputPatterns = backendCfg.files || DEFAULT_FILES
-  const appYamlPassEnv = opt.appYamlPassEnv || backendCfg.appYamlPassEnv
-
-  console.log(`1. Copy files to ${dimGrey(targetDir)}`)
-
-  // Clean targetDir
-  fs2.emptyDir(targetDir)
-
-  kpySync({
-    baseDir: defaultFilesDir,
-    outputDir: targetDir,
-    dotfiles: true,
-  })
-
-  kpySync({
-    baseDir: projectDir,
-    inputPatterns,
-    outputDir: targetDir,
-    dotfiles: true,
-  })
-
-  const { NPM_TOKEN } = process.env
-  if (NPM_TOKEN && createNpmrc) {
-    const npmrcPath = `${targetDir}/.npmrc`
-    const npmrc = `//registry.npmjs.org/:_authToken=${NPM_TOKEN}`
-    fs2.writeFile(npmrcPath, npmrc)
-  }
-
-  console.log(`2. Generate ${dimGrey('deployInfo.json')} and ${dimGrey('app.yaml')} in targetDir`)
-
-  const deployInfo = await createAndSaveDeployInfo(backendCfg, targetDir)
-  createAndSaveAppYaml(backendCfg, deployInfo, projectDir, targetDir, appYamlPassEnv)
-
-  return deployInfo
-}

package/src/deploy/files-default/.gcloudignore

@@ -1,7 +0,0 @@
-#
-# .gcloudignore
-#
-.gcloudignore
-.git
-.gitignore
-node_modules/