@naturalcycles/backend-lib 9.78.0 → 9.80.0

package/dist/deploy/deployGae.js

@@ -1,90 +0,0 @@
-import { _anyToError } from '@naturalcycles/js-lib/error';
-import { pRetry } from '@naturalcycles/js-lib/promise/pRetry.js';
-import { _objectAssign } from '@naturalcycles/js-lib/types';
-import { appendToGithubSummary } from '@naturalcycles/nodejs-lib/env';
-import { exec2 } from '@naturalcycles/nodejs-lib/exec2';
-import { fs2 } from '@naturalcycles/nodejs-lib/fs2';
-import { getBackendCfg } from './backend.cfg.util.js';
-import { createDeployInfo } from './deploy.util.js';
-import { deployHealthCheck } from './deployHealthCheck.js';
-import { deployPrepare } from './deployPrepare.js';
-export async function deployGae(opt = {}) {
-    const { logOnFailure, logOnSuccess } = opt;
-    // 1. build
-    const packageManager = detectPackageManager();
-    exec2.spawn(`${packageManager} build`);
-    // 2. deploy-prepare
-    const deployInfo = await deployPrepare();
-    const targetDir = './tmp/deploy';
-    const appYamlPath = `${targetDir}/app.yaml`;
-    const { gaeProject, gaeService, gaeVersion, versionUrl, serviceUrl } = deployInfo;
-    _objectAssign(opt, {
-        gaeProject,
-        gaeService,
-        gaeVersion,
-    });
-    await pRetry(async () => {
-        try {
-            exec2.spawn(`gcloud app deploy ${appYamlPath} --project ${gaeProject} --version ${gaeVersion} --quiet --no-promote`);
-        }
-        catch (err) {
-            if (logOnFailure) {
-                logs(gaeProject, gaeService, gaeVersion);
-            }
-            throw err;
-        }
-    }, {
-        name: 'deploy',
-        maxAttempts: 2,
-        delay: 30_000,
-        // todo: this doesn't work, as the error is different from what is logged.
-        // We shoud somehow capture the logged text
-        predicate: err => _anyToError(err).message.includes('operation is already in progress'),
-    });
-    // Health check (versionUrl)
-    // yarn deploy-health-check --url $deployInfo_versionUrl --repeat 3 --timeoutSec 180 --intervalSec 2
-    await deployHealthCheck(versionUrl, opt);
-    // Only if "timestamped version" is used ('1' is default)
-    if (gaeVersion !== '1') {
-        // Rollout (promote versionUrl to serviceUrl)
-        // gcloud app services set-traffic $deployInfo_gaeService --project $deployInfo_gaeProject --splits $deployInfo_gaeVersion=1 --quiet
-        exec2.spawn(`gcloud app services set-traffic ${gaeService} --project ${gaeProject} --splits ${gaeVersion}=1 --quiet`);
-        // Health check (serviceUrl)
-        // yarn deploy-health-check --url $deployInfo_serviceUrl --repeat 3 --timeoutSec 60 --intervalSec 2
-        await deployHealthCheck(serviceUrl, opt);
-    }
-    appendToGithubSummary(`Deployed ${serviceUrl}`);
-    // Logs
-    if (logOnSuccess) {
-        logs(gaeProject, gaeService, gaeVersion);
-    }
-}
-/**
- * Undeploys/removes the GAE service/version, using the same rules as deployGae.
- * Detects the service/version by the same criteria: branch name, backend.cfg.yaml, etc.
- */
-export async function undeployGae(branch) {
-    const projectDir = '.';
-    const backendCfg = getBackendCfg(projectDir);
-    const { gaeProject, gaeService, gaeVersion } = await createDeployInfo(backendCfg, branch);
-    // if (prod) {
-    //   console.log(`undeployGae (branch: ${branch}): not removing prod version (safety check)`)
-    //   return
-    // }
-    console.log(`undeployGae (branch: ${branch}): going to remove ${gaeProject}/${gaeService}/${gaeVersion}`);
-    exec2.spawn(`gcloud app versions delete --project ${gaeProject} --service ${gaeService} ${gaeVersion} --quiet`);
-    appendToGithubSummary(`removed ${gaeProject}/${gaeService}/${gaeVersion}`);
-}
-function logs(gaeProject, gaeService, gaeVersion) {
-    try {
-        exec2.spawn(`gcloud app logs read --project ${gaeProject} --service ${gaeService} --version ${gaeVersion}`);
-    }
-    catch { }
-}
-function detectPackageManager() {
-    const pj = fs2.readText('package.json');
-    if (pj.includes('pnpm')) {
-        return 'pnpm';
-    }
-    return 'yarn'; // yarn is still the default
-}

package/dist/deploy/deployInfo.util.d.ts

@@ -1 +0,0 @@
-export declare const getDeployInfo: any;

package/dist/deploy/deployInfo.util.js

@@ -1,25 +0,0 @@
-import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js';
-import { _memoized } from '@naturalcycles/js-lib/decorators';
-import { fs2 } from '@naturalcycles/nodejs-lib/fs2';
-export const getDeployInfo = _memoized((projectDir) => {
-    const deployInfoPath = `${projectDir}/deployInfo.json`;
-    try {
-        return fs2.readJson(deployInfoPath);
-    }
-    catch {
-        // console.error(`cannot read ${deployInfoPath}, returning empty version`)
-        return getDeployInfoStub();
-    }
-});
-function getDeployInfoStub(stub = '') {
-    return {
-        gaeProject: stub,
-        gaeService: stub,
-        gaeVersion: stub,
-        serviceUrl: stub,
-        versionUrl: stub,
-        gitBranch: stub,
-        gitRev: stub,
-        ts: localTime.nowUnix(),
-    };
-}

package/dist/deploy/deployPrepare.d.ts

@@ -1,13 +0,0 @@
-import type { DeployInfo } from './deploy.model.js';
-export interface DeployPrepareOptions {
-    projectDir?: string;
-    targetDir?: string;
-    createNpmrc?: boolean;
-    /**
-     * Comma-separated list of env variables that will be passed to app.yaml from process.env.
-     * Use it to pass secrets.
-     */
-    appYamlPassEnv?: string;
-}
-export declare const deployPrepareCliOptions: CliOptions;
-export declare function deployPrepare(opt?: DeployPrepareOptions): Promise<DeployInfo>;

package/dist/deploy/deployPrepare.js

@@ -1,85 +0,0 @@
-import { loadEnvFileIfExists } from '@naturalcycles/nodejs-lib';
-import { dimGrey } from '@naturalcycles/nodejs-lib/colors';
-import { fs2 } from '@naturalcycles/nodejs-lib/fs2';
-import { kpySync } from '@naturalcycles/nodejs-lib/kpy';
-import { srcDir } from '../paths.cnst.js';
-import { getBackendCfg } from './backend.cfg.util.js';
-import { createAndSaveAppYaml, createAndSaveDeployInfo } from './deploy.util.js';
-export const deployPrepareCliOptions = {
-    projectDir: {
-        type: 'string',
-        default: '.',
-    },
-    targetDir: {
-        type: 'string',
-        default: `./tmp/deploy`,
-    },
-    createNpmrc: {
-        type: 'boolean',
-        desc: 'Create .npmrc in targetDir if NPM_TOKEN env var is set',
-        default: true,
-    },
-    appYamlPassEnv: {
-        type: 'string',
-        desc: 'Comma-separated list of env variables that will be passed to app.yaml from process.env',
-    },
-};
-const DEFAULT_FILES = [
-    'dist',
-    // '!dist/test',
-    // '!dist/**/*.test.*',
-    // '!dist/**/*.mock.*',
-    '!dist/**/*.script.*',
-    'src', // For Sentry
-    '!src/test',
-    '!src/**/*.test.*',
-    '!src/**/*.mock.*',
-    '!src/**/*.script.*',
-    '!**/__snapshots__',
-    '!**/__exclude',
-    'static',
-    'secret/**/*.enc', // encrypted secrets
-    'package.json',
-    'yarn.lock',
-    '.yarnrc',
-    'pnpm-lock.yaml',
-    'pnpm-workspace.yaml',
-    // 'tsconfig.json', // for path-mapping to work!
-    // 'tsconfig.dist.json',
-    '.gcloudignore',
-    'app.yaml',
-    'patches', // to allow patch-package
-    'resources',
-];
-const defaultFilesDir = `${srcDir}/deploy/files-default`;
-export async function deployPrepare(opt = {}) {
-    loadEnvFileIfExists();
-    const { projectDir = '.', targetDir = './tmp/deploy', createNpmrc = true } = opt;
-    const backendCfg = getBackendCfg(projectDir);
-    const inputPatterns = backendCfg.files || DEFAULT_FILES;
-    const appYamlPassEnv = opt.appYamlPassEnv || backendCfg.appYamlPassEnv;
-    console.log(`1. Copy files to ${dimGrey(targetDir)}`);
-    // Clean targetDir
-    fs2.emptyDir(targetDir);
-    kpySync({
-        baseDir: defaultFilesDir,
-        outputDir: targetDir,
-        dotfiles: true,
-    });
-    kpySync({
-        baseDir: projectDir,
-        inputPatterns,
-        outputDir: targetDir,
-        dotfiles: true,
-    });
-    const { NPM_TOKEN } = process.env;
-    if (NPM_TOKEN && createNpmrc) {
-        const npmrcPath = `${targetDir}/.npmrc`;
-        const npmrc = `//registry.npmjs.org/:_authToken=${NPM_TOKEN}`;
-        fs2.writeFile(npmrcPath, npmrc);
-    }
-    console.log(`2. Generate ${dimGrey('deployInfo.json')} and ${dimGrey('app.yaml')} in targetDir`);
-    const deployInfo = await createAndSaveDeployInfo(backendCfg, targetDir);
-    createAndSaveAppYaml(backendCfg, deployInfo, projectDir, targetDir, appYamlPassEnv);
-    return deployInfo;
-}

package/src/bin/deploy-gae.ts

@@ -1,14 +0,0 @@
-import { _parseArgs } from '@naturalcycles/nodejs-lib/args'
-import { runScript } from '@naturalcycles/nodejs-lib/runScript'
-import { deployGae } from '../deploy/deployGae.js'
-import { deployHealthCheckCliOptions } from '../deploy/deployHealthCheck.js'
-import { deployPrepareCliOptions } from '../deploy/deployPrepare.js'
-
-runScript(async () => {
-  const opt = _parseArgs({
-    ...deployPrepareCliOptions,
-    ...deployHealthCheckCliOptions,
-  })
-
-  await deployGae(opt)
-})

package/src/bin/deploy-prepare.ts

@@ -1,24 +0,0 @@
-/*
-
-yarn deploy-prepare
-
- */
-
-import { _parseArgs } from '@naturalcycles/nodejs-lib/args'
-import { runScript } from '@naturalcycles/nodejs-lib/runScript'
-import { deployPrepare, deployPrepareCliOptions } from '../deploy/deployPrepare.js'
-
-runScript(async () => {
-  const opt = _parseArgs(deployPrepareCliOptions)
-
-  await deployPrepare(opt)
-})
-
-// deploy strategy
-// gae project: from config
-// gae service: from config -- branch name
-// gae version: automatic form date
-
-// yarn build-prod
-// yarn deploy-prepare && ./tmp/deploy/app.yaml && ./tmp/deploy/deployInfo.json && json2env
-//

package/src/bin/undeploy-gae.ts

@@ -1,15 +0,0 @@
-import { _parseArgs } from '@naturalcycles/nodejs-lib/args'
-import { runScript } from '@naturalcycles/nodejs-lib/runScript'
-import { undeployGae } from '../deploy/deployGae.js'
-
-runScript(async () => {
-  const { branch } = _parseArgs({
-    branch: {
-      type: 'string',
-      demandOption: true,
-      desc: `Because Github Actions delete event happens after the branch is already deleted - you need to pass it manually`,
-    },
-  })
-
-  await undeployGae(branch)
-})

package/src/deploy/backend.cfg.util.ts

@@ -1,63 +0,0 @@
-import type { StringMap } from '@naturalcycles/js-lib/types'
-import { j } from '@naturalcycles/nodejs-lib/ajv'
-import { fs2 } from '@naturalcycles/nodejs-lib/fs2'
-import { yaml2 } from '@naturalcycles/nodejs-lib/yaml2'
-
-export interface BackendCfg {
-  gaeProject: string
-  gaeProjectByBranch?: StringMap
-
-  /**
-   * @example default
-   */
-  gaeService: string
-  gaeServiceByBranch?: StringMap
-
-  /**
-   * List of file patterns to include in deployment.
-   */
-  files?: string[]
-
-  appEnvDefault: string
-  appEnvByBranch?: StringMap
-
-  /**
-   * List of branches to use timestamps in gae version names (to keep previous versions).
-   */
-  branchesWithTimestampVersions?: string[]
-
-  /**
-   * If true - branch names are not passed into deployed urls as is, but are hashed.
-   */
-  hashedBranches?: boolean
-
-  /**
-   * Comma-separated list of env variables that will be passed to app.yaml from process.env
-   */
-  appYamlPassEnv?: string
-}
-
-export const backendCfgSchema = j.object<BackendCfg>({
-  gaeProject: j.string(),
-  gaeProjectByBranch: j.object.stringMap(j.string()).optional(),
-  gaeService: j.string(),
-  gaeServiceByBranch: j.object.stringMap(j.string()).optional(),
-  files: j.array(j.string()).optional(),
-  appEnvDefault: j.string(),
-  appEnvByBranch: j.object.stringMap(j.string()).optional(),
-  branchesWithTimestampVersions: j.array(j.string()).optional(),
-  hashedBranches: j.boolean().optional(),
-  appYamlPassEnv: j.string().optional(),
-})
-
-export function getBackendCfg(projectDir = '.'): BackendCfg {
-  const backendCfgYamlPath = `${projectDir}/backend.cfg.yaml`
-
-  fs2.requireFileToExist(backendCfgYamlPath)
-
-  const backendCfg: BackendCfg = {
-    ...yaml2.readYaml(backendCfgYamlPath),
-  }
-
-  return backendCfgSchema.validate(backendCfg)
-}

package/src/deploy/deploy.model.ts

@@ -1,32 +0,0 @@
-import type { UnixTimestamp } from '@naturalcycles/js-lib/types'
-
-export interface DeployInfo {
-  gaeProject: string
-  gaeService: string
-  gaeVersion: string
-  versionUrl: string
-  serviceUrl: string
-  gitRev: string
-  gitBranch: string
-
-  /**
-   * Unix timestamp of deployInfo.json being generated.
-   */
-  ts: UnixTimestamp
-}
-
-export interface AppYaml extends Record<string, any> {
-  runtime: string
-  service: string
-  inbound_services?: string[]
-  instance_class?: string
-  automatic_scaling?: {
-    min_instances?: number
-    max_instances?: number
-  }
-  env_variables: {
-    APP_ENV: string
-    DEBUG: string
-    [k: string]: string
-  }
-}

package/src/deploy/deploy.util.ts

@@ -1,214 +0,0 @@
-import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js'
-import { _assert } from '@naturalcycles/js-lib/error/assert.js'
-import { _mapValues, _merge } from '@naturalcycles/js-lib/object/object.util.js'
-import { _truncate } from '@naturalcycles/js-lib/string/string.util.js'
-import { sha256 } from '@naturalcycles/nodejs-lib'
-import { dimGrey, white } from '@naturalcycles/nodejs-lib/colors'
-import { fs2 } from '@naturalcycles/nodejs-lib/fs2'
-import { yaml2 } from '@naturalcycles/nodejs-lib/yaml2'
-import type { BackendCfg } from './backend.cfg.util.js'
-import type { AppYaml, DeployInfo } from './deploy.model.js'
-
-const APP_YAML_DEFAULT = (): AppYaml => ({
-  runtime: 'nodejs22',
-  service: 'default',
-  inbound_services: ['warmup'],
-  instance_class: 'F1',
-  automatic_scaling: {
-    min_instances: 0,
-    max_instances: 1,
-  },
-  env_variables: {
-    APP_ENV: 'prod',
-    DEBUG: 'app*,nc:*',
-    DEBUG_HIDE_DATE: 'true',
-    // DEBUG_COLORS: 'true',
-    TZ: 'UTC',
-  },
-})
-
-export async function createAndSaveDeployInfo(
-  backendCfg: BackendCfg,
-  targetDir: string,
-): Promise<DeployInfo> {
-  const deployInfo = await createDeployInfo(backendCfg)
-  const deployInfoPath = `${targetDir}/deployInfo.json`
-  fs2.writeJson(deployInfoPath, deployInfo, { spaces: 2 })
-  console.log(`saved ${dimGrey(deployInfoPath)}`)
-  return deployInfo
-}
-
-export async function createDeployInfo(
-  backendCfg: BackendCfg,
-  overrideBranch?: string,
-): Promise<DeployInfo> {
-  const { simpleGit } = await import('simple-git') // lazy load
-  const git = simpleGit('.')
-
-  const now = localTime.now()
-  const gitBranch = overrideBranch || (await git.status()).current!
-  const gitRev = (await git.revparse(['HEAD'])).slice(0, 7)
-
-  let {
-    gaeProject,
-    gaeProjectByBranch = {},
-    gaeService,
-    gaeServiceByBranch = {},
-    branchesWithTimestampVersions = [],
-  } = backendCfg
-
-  gaeProject = gaeProjectByBranch[gitBranch] || gaeProject
-
-  if (gaeServiceByBranch[gitBranch]) {
-    gaeService = validateGAEServiceName(gaeServiceByBranch[gitBranch])
-  } else {
-    let branchName = gitBranch
-
-    if (backendCfg.hashedBranches) {
-      // Obfuscates the branch name by hashing it.
-      // If there are Jira issue names in the branch name, the first one found will be used as a prefix.
-      const jiraIssue = gitBranch.match(/([Dd][Ee][Vv]-\d+)/)?.[0]
-      const branchHash = sha256(gitBranch).slice(0, 10)
-      branchName = [jiraIssue, branchHash].filter(Boolean).join('-')
-    }
-
-    gaeService = validateGAEServiceName([branchName, gaeService].join('--'))
-  }
-
-  let gaeVersion = '1'
-
-  if (branchesWithTimestampVersions.includes(gitBranch)) {
-    // May only contain lowercase letters, digits, and hyphens. Must begin and end with a letter or digit. Must not exceed 63 characters.
-    gaeVersion = [
-      // now.format('YYYYMMDD-HHmm'), // 20190521-1721
-      now.toStringCompact().replace('_', '-'),
-      gitRev,
-    ].join('-')
-  }
-
-  const versionUrl = `https://${[gaeVersion, gaeService, gaeProject].join('-dot-')}.appspot.com`
-
-  // Check the 63-char limit
-  const versionUrlString = [gaeVersion, gaeService, gaeProject].join('-dot-')
-  _assert(
-    versionUrlString.length <= 63,
-    `versionUrl length should be <= 63 characters, but it's ${versionUrlString.length} instead: ${versionUrlString}`,
-  )
-
-  const serviceUrl = `https://${[gaeService, gaeProject].join('-dot-')}.appspot.com`
-
-  const deployInfo: DeployInfo = {
-    gaeProject,
-    gaeService,
-    gaeVersion,
-    versionUrl,
-    serviceUrl,
-    gitBranch,
-    gitRev,
-    ts: now.unix,
-  }
-
-  console.log({ deployInfo })
-
-  return deployInfo
-}
-
-export function createAndSaveAppYaml(
-  backendCfg: BackendCfg,
-  deployInfo: DeployInfo,
-  projectDir: string,
-  targetDir: string,
-  appYamlPassEnv = '',
-): AppYaml {
-  const appYaml = createAppYaml(backendCfg, deployInfo, projectDir, appYamlPassEnv)
-  const appYamlPath = `${targetDir}/app.yaml`
-  yaml2.writeYaml(appYamlPath, appYaml)
-  console.log(`saved ${dimGrey(appYamlPath)}`)
-  return appYaml
-}
-
-export function createAppYaml(
-  backendCfg: BackendCfg,
-  deployInfo: DeployInfo,
-  projectDir: string,
-  appYamlPassEnv = '',
-): AppYaml {
-  const { appEnvDefault, appEnvByBranch = {} } = backendCfg
-  const { gaeService: service, gitBranch } = deployInfo
-
-  const { APP_ENV: processAppEnv } = process.env
-  const APP_ENV = processAppEnv || appEnvByBranch[gitBranch] || appEnvDefault
-  if (processAppEnv) {
-    console.log(`using APP_ENV=${dimGrey(processAppEnv)} from process.env`)
-  }
-
-  const appYaml = APP_YAML_DEFAULT()
-
-  // Check existing app.yaml
-  const appYamlPath = `${projectDir}/app.yaml`
-  if (fs2.pathExists(appYamlPath)) {
-    console.log(`merging-in ${dimGrey(appYamlPath)}`)
-    _merge(appYaml, yaml2.readYaml(appYamlPath))
-  }
-
-  const appEnvYamlPath = `${projectDir}/app.${APP_ENV}.yaml`
-  if (fs2.pathExists(appEnvYamlPath)) {
-    console.log(`merging-in ${dimGrey(appEnvYamlPath)}`)
-    _merge(appYaml, yaml2.readYaml(appEnvYamlPath))
-  }
-
-  // appYamlPassEnv
-  const passEnv = appYamlPassEnv
-    .split(',')
-    .filter(Boolean)
-    // oxlint-disable-next-line unicorn/no-array-reduce
-    .reduce<Record<string, string>>((map, key) => {
-      const v = process.env[key]
-      if (!v) {
-        throw new Error(
-          `appYamlPassEnv.${key} is requested, but process.env.${key} is not defined!`,
-        )
-      }
-      map[key] = v
-      return map
-    }, {})
-
-  if (Object.keys(passEnv).length) {
-    console.log(
-      `will merge ${white(
-        String(Object.keys(passEnv).length),
-      )} process.env keys to app.yaml: ${dimGrey(Object.keys(passEnv).join(', '))}`,
-    )
-  }
-
-  _merge(appYaml, {
-    service,
-    env_variables: {
-      APP_ENV,
-      ...passEnv,
-    },
-  })
-
-  // Redacted appYaml to not show up secrets
-  console.log({
-    appYaml: redactedAppYaml(appYaml),
-  })
-
-  return appYaml
-}
-
-function redactedAppYaml(appYaml: AppYaml): AppYaml {
-  return {
-    ...appYaml,
-    env_variables: _mapValues(appYaml.env_variables || {}, (_k, v) => _truncate(v, 7)),
-  }
-}
-
-export function validateGAEServiceName(serviceName: string): string {
-  // May only contain lowercase letters, digits, and hyphens. Must begin and end with a letter or digit. Must not exceed 63 characters.
-  return serviceName
-    .replaceAll('_', '-')
-    .toLowerCase()
-    .replaceAll(/[^0-9a-z-]/gi, '')
-    .slice(0, 40)
-}