@naturalcycles/backend-lib 9.79.0 → 9.80.0

package/dist/deploy/deploy.util.js

@@ -1,151 +0,0 @@
-import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js';
-import { _assert } from '@naturalcycles/js-lib/error/assert.js';
-import { _mapValues, _merge } from '@naturalcycles/js-lib/object/object.util.js';
-import { _truncate } from '@naturalcycles/js-lib/string/string.util.js';
-import { sha256 } from '@naturalcycles/nodejs-lib';
-import { dimGrey, white } from '@naturalcycles/nodejs-lib/colors';
-import { fs2 } from '@naturalcycles/nodejs-lib/fs2';
-import { yaml2 } from '@naturalcycles/nodejs-lib/yaml2';
-const APP_YAML_DEFAULT = () => ({
-    runtime: 'nodejs22',
-    service: 'default',
-    inbound_services: ['warmup'],
-    instance_class: 'F1',
-    automatic_scaling: {
-        min_instances: 0,
-        max_instances: 1,
-    },
-    env_variables: {
-        APP_ENV: 'prod',
-        DEBUG: 'app*,nc:*',
-        DEBUG_HIDE_DATE: 'true',
-        // DEBUG_COLORS: 'true',
-        TZ: 'UTC',
-    },
-});
-export async function createAndSaveDeployInfo(backendCfg, targetDir) {
-    const deployInfo = await createDeployInfo(backendCfg);
-    const deployInfoPath = `${targetDir}/deployInfo.json`;
-    fs2.writeJson(deployInfoPath, deployInfo, { spaces: 2 });
-    console.log(`saved ${dimGrey(deployInfoPath)}`);
-    return deployInfo;
-}
-export async function createDeployInfo(backendCfg, overrideBranch) {
-    const { simpleGit } = await import('simple-git'); // lazy load
-    const git = simpleGit('.');
-    const now = localTime.now();
-    const gitBranch = overrideBranch || (await git.status()).current;
-    const gitRev = (await git.revparse(['HEAD'])).slice(0, 7);
-    let { gaeProject, gaeProjectByBranch = {}, gaeService, gaeServiceByBranch = {}, branchesWithTimestampVersions = [], } = backendCfg;
-    gaeProject = gaeProjectByBranch[gitBranch] || gaeProject;
-    if (gaeServiceByBranch[gitBranch]) {
-        gaeService = validateGAEServiceName(gaeServiceByBranch[gitBranch]);
-    }
-    else {
-        let branchName = gitBranch;
-        if (backendCfg.hashedBranches) {
-            // Obfuscates the branch name by hashing it.
-            // If there are Jira issue names in the branch name, the first one found will be used as a prefix.
-            const jiraIssue = gitBranch.match(/([Dd][Ee][Vv]-\d+)/)?.[0];
-            const branchHash = sha256(gitBranch).slice(0, 10);
-            branchName = [jiraIssue, branchHash].filter(Boolean).join('-');
-        }
-        gaeService = validateGAEServiceName([branchName, gaeService].join('--'));
-    }
-    let gaeVersion = '1';
-    if (branchesWithTimestampVersions.includes(gitBranch)) {
-        // May only contain lowercase letters, digits, and hyphens. Must begin and end with a letter or digit. Must not exceed 63 characters.
-        gaeVersion = [
-            // now.format('YYYYMMDD-HHmm'), // 20190521-1721
-            now.toStringCompact().replace('_', '-'),
-            gitRev,
-        ].join('-');
-    }
-    const versionUrl = `https://${[gaeVersion, gaeService, gaeProject].join('-dot-')}.appspot.com`;
-    // Check the 63-char limit
-    const versionUrlString = [gaeVersion, gaeService, gaeProject].join('-dot-');
-    _assert(versionUrlString.length <= 63, `versionUrl length should be <= 63 characters, but it's ${versionUrlString.length} instead: ${versionUrlString}`);
-    const serviceUrl = `https://${[gaeService, gaeProject].join('-dot-')}.appspot.com`;
-    const deployInfo = {
-        gaeProject,
-        gaeService,
-        gaeVersion,
-        versionUrl,
-        serviceUrl,
-        gitBranch,
-        gitRev,
-        ts: now.unix,
-    };
-    console.log({ deployInfo });
-    return deployInfo;
-}
-export function createAndSaveAppYaml(backendCfg, deployInfo, projectDir, targetDir, appYamlPassEnv = '') {
-    const appYaml = createAppYaml(backendCfg, deployInfo, projectDir, appYamlPassEnv);
-    const appYamlPath = `${targetDir}/app.yaml`;
-    yaml2.writeYaml(appYamlPath, appYaml);
-    console.log(`saved ${dimGrey(appYamlPath)}`);
-    return appYaml;
-}
-export function createAppYaml(backendCfg, deployInfo, projectDir, appYamlPassEnv = '') {
-    const { appEnvDefault, appEnvByBranch = {} } = backendCfg;
-    const { gaeService: service, gitBranch } = deployInfo;
-    const { APP_ENV: processAppEnv } = process.env;
-    const APP_ENV = processAppEnv || appEnvByBranch[gitBranch] || appEnvDefault;
-    if (processAppEnv) {
-        console.log(`using APP_ENV=${dimGrey(processAppEnv)} from process.env`);
-    }
-    const appYaml = APP_YAML_DEFAULT();
-    // Check existing app.yaml
-    const appYamlPath = `${projectDir}/app.yaml`;
-    if (fs2.pathExists(appYamlPath)) {
-        console.log(`merging-in ${dimGrey(appYamlPath)}`);
-        _merge(appYaml, yaml2.readYaml(appYamlPath));
-    }
-    const appEnvYamlPath = `${projectDir}/app.${APP_ENV}.yaml`;
-    if (fs2.pathExists(appEnvYamlPath)) {
-        console.log(`merging-in ${dimGrey(appEnvYamlPath)}`);
-        _merge(appYaml, yaml2.readYaml(appEnvYamlPath));
-    }
-    // appYamlPassEnv
-    const passEnv = appYamlPassEnv
-        .split(',')
-        .filter(Boolean)
-        // oxlint-disable-next-line unicorn/no-array-reduce
-        .reduce((map, key) => {
-        const v = process.env[key];
-        if (!v) {
-            throw new Error(`appYamlPassEnv.${key} is requested, but process.env.${key} is not defined!`);
-        }
-        map[key] = v;
-        return map;
-    }, {});
-    if (Object.keys(passEnv).length) {
-        console.log(`will merge ${white(String(Object.keys(passEnv).length))} process.env keys to app.yaml: ${dimGrey(Object.keys(passEnv).join(', '))}`);
-    }
-    _merge(appYaml, {
-        service,
-        env_variables: {
-            APP_ENV,
-            ...passEnv,
-        },
-    });
-    // Redacted appYaml to not show up secrets
-    console.log({
-        appYaml: redactedAppYaml(appYaml),
-    });
-    return appYaml;
-}
-function redactedAppYaml(appYaml) {
-    return {
-        ...appYaml,
-        env_variables: _mapValues(appYaml.env_variables || {}, (_k, v) => _truncate(v, 7)),
-    };
-}
-export function validateGAEServiceName(serviceName) {
-    // May only contain lowercase letters, digits, and hyphens. Must begin and end with a letter or digit. Must not exceed 63 characters.
-    return serviceName
-        .replaceAll('_', '-')
-        .toLowerCase()
-        .replaceAll(/[^0-9a-z-]/gi, '')
-        .slice(0, 40);
-}

package/dist/deploy/deployGae.d.ts

@@ -1,10 +0,0 @@
-import type { DeployHealthCheckOptions } from './deployHealthCheck.js';
-import type { DeployPrepareOptions } from './deployPrepare.js';
-export interface DeployGaeOptions extends DeployPrepareOptions, DeployHealthCheckOptions {
-}
-export declare function deployGae(opt?: DeployGaeOptions): Promise<void>;
-/**
- * Undeploys/removes the GAE service/version, using the same rules as deployGae.
- * Detects the service/version by the same criteria: branch name, backend.cfg.yaml, etc.
- */
-export declare function undeployGae(branch: string): Promise<void>;

package/dist/deploy/deployGae.js

@@ -1,90 +0,0 @@
-import { _anyToError } from '@naturalcycles/js-lib/error';
-import { pRetry } from '@naturalcycles/js-lib/promise/pRetry.js';
-import { _objectAssign } from '@naturalcycles/js-lib/types';
-import { appendToGithubSummary } from '@naturalcycles/nodejs-lib/env';
-import { exec2 } from '@naturalcycles/nodejs-lib/exec2';
-import { fs2 } from '@naturalcycles/nodejs-lib/fs2';
-import { getBackendCfg } from './backend.cfg.util.js';
-import { createDeployInfo } from './deploy.util.js';
-import { deployHealthCheck } from './deployHealthCheck.js';
-import { deployPrepare } from './deployPrepare.js';
-export async function deployGae(opt = {}) {
-    const { logOnFailure, logOnSuccess } = opt;
-    // 1. build
-    const packageManager = detectPackageManager();
-    exec2.spawn(`${packageManager} build`);
-    // 2. deploy-prepare
-    const deployInfo = await deployPrepare();
-    const targetDir = './tmp/deploy';
-    const appYamlPath = `${targetDir}/app.yaml`;
-    const { gaeProject, gaeService, gaeVersion, versionUrl, serviceUrl } = deployInfo;
-    _objectAssign(opt, {
-        gaeProject,
-        gaeService,
-        gaeVersion,
-    });
-    await pRetry(async () => {
-        try {
-            exec2.spawn(`gcloud app deploy ${appYamlPath} --project ${gaeProject} --version ${gaeVersion} --quiet --no-promote`);
-        }
-        catch (err) {
-            if (logOnFailure) {
-                logs(gaeProject, gaeService, gaeVersion);
-            }
-            throw err;
-        }
-    }, {
-        name: 'deploy',
-        maxAttempts: 2,
-        delay: 30_000,
-        // todo: this doesn't work, as the error is different from what is logged.
-        // We shoud somehow capture the logged text
-        predicate: err => _anyToError(err).message.includes('operation is already in progress'),
-    });
-    // Health check (versionUrl)
-    // yarn deploy-health-check --url $deployInfo_versionUrl --repeat 3 --timeoutSec 180 --intervalSec 2
-    await deployHealthCheck(versionUrl, opt);
-    // Only if "timestamped version" is used ('1' is default)
-    if (gaeVersion !== '1') {
-        // Rollout (promote versionUrl to serviceUrl)
-        // gcloud app services set-traffic $deployInfo_gaeService --project $deployInfo_gaeProject --splits $deployInfo_gaeVersion=1 --quiet
-        exec2.spawn(`gcloud app services set-traffic ${gaeService} --project ${gaeProject} --splits ${gaeVersion}=1 --quiet`);
-        // Health check (serviceUrl)
-        // yarn deploy-health-check --url $deployInfo_serviceUrl --repeat 3 --timeoutSec 60 --intervalSec 2
-        await deployHealthCheck(serviceUrl, opt);
-    }
-    appendToGithubSummary(`Deployed ${serviceUrl}`);
-    // Logs
-    if (logOnSuccess) {
-        logs(gaeProject, gaeService, gaeVersion);
-    }
-}
-/**
- * Undeploys/removes the GAE service/version, using the same rules as deployGae.
- * Detects the service/version by the same criteria: branch name, backend.cfg.yaml, etc.
- */
-export async function undeployGae(branch) {
-    const projectDir = '.';
-    const backendCfg = getBackendCfg(projectDir);
-    const { gaeProject, gaeService, gaeVersion } = await createDeployInfo(backendCfg, branch);
-    // if (prod) {
-    //   console.log(`undeployGae (branch: ${branch}): not removing prod version (safety check)`)
-    //   return
-    // }
-    console.log(`undeployGae (branch: ${branch}): going to remove ${gaeProject}/${gaeService}/${gaeVersion}`);
-    exec2.spawn(`gcloud app versions delete --project ${gaeProject} --service ${gaeService} ${gaeVersion} --quiet`);
-    appendToGithubSummary(`removed ${gaeProject}/${gaeService}/${gaeVersion}`);
-}
-function logs(gaeProject, gaeService, gaeVersion) {
-    try {
-        exec2.spawn(`gcloud app logs read --project ${gaeProject} --service ${gaeService} --version ${gaeVersion}`);
-    }
-    catch { }
-}
-function detectPackageManager() {
-    const pj = fs2.readText('package.json');
-    if (pj.includes('pnpm')) {
-        return 'pnpm';
-    }
-    return 'yarn'; // yarn is still the default
-}

package/dist/deploy/deployInfo.util.d.ts

@@ -1 +0,0 @@
-export declare const getDeployInfo: any;

package/dist/deploy/deployInfo.util.js

@@ -1,25 +0,0 @@
-import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js';
-import { _memoized } from '@naturalcycles/js-lib/decorators';
-import { fs2 } from '@naturalcycles/nodejs-lib/fs2';
-export const getDeployInfo = _memoized((projectDir) => {
-    const deployInfoPath = `${projectDir}/deployInfo.json`;
-    try {
-        return fs2.readJson(deployInfoPath);
-    }
-    catch {
-        // console.error(`cannot read ${deployInfoPath}, returning empty version`)
-        return getDeployInfoStub();
-    }
-});
-function getDeployInfoStub(stub = '') {
-    return {
-        gaeProject: stub,
-        gaeService: stub,
-        gaeVersion: stub,
-        serviceUrl: stub,
-        versionUrl: stub,
-        gitBranch: stub,
-        gitRev: stub,
-        ts: localTime.nowUnix(),
-    };
-}

package/dist/deploy/deployPrepare.d.ts

@@ -1,13 +0,0 @@
-import type { DeployInfo } from './deploy.model.js';
-export interface DeployPrepareOptions {
-    projectDir?: string;
-    targetDir?: string;
-    createNpmrc?: boolean;
-    /**
-     * Comma-separated list of env variables that will be passed to app.yaml from process.env.
-     * Use it to pass secrets.
-     */
-    appYamlPassEnv?: string;
-}
-export declare const deployPrepareCliOptions: CliOptions;
-export declare function deployPrepare(opt?: DeployPrepareOptions): Promise<DeployInfo>;

package/dist/deploy/deployPrepare.js

@@ -1,85 +0,0 @@
-import { loadEnvFileIfExists } from '@naturalcycles/nodejs-lib';
-import { dimGrey } from '@naturalcycles/nodejs-lib/colors';
-import { fs2 } from '@naturalcycles/nodejs-lib/fs2';
-import { kpySync } from '@naturalcycles/nodejs-lib/kpy';
-import { srcDir } from '../paths.cnst.js';
-import { getBackendCfg } from './backend.cfg.util.js';
-import { createAndSaveAppYaml, createAndSaveDeployInfo } from './deploy.util.js';
-export const deployPrepareCliOptions = {
-    projectDir: {
-        type: 'string',
-        default: '.',
-    },
-    targetDir: {
-        type: 'string',
-        default: `./tmp/deploy`,
-    },
-    createNpmrc: {
-        type: 'boolean',
-        desc: 'Create .npmrc in targetDir if NPM_TOKEN env var is set',
-        default: true,
-    },
-    appYamlPassEnv: {
-        type: 'string',
-        desc: 'Comma-separated list of env variables that will be passed to app.yaml from process.env',
-    },
-};
-const DEFAULT_FILES = [
-    'dist',
-    // '!dist/test',
-    // '!dist/**/*.test.*',
-    // '!dist/**/*.mock.*',
-    '!dist/**/*.script.*',
-    'src', // For Sentry
-    '!src/test',
-    '!src/**/*.test.*',
-    '!src/**/*.mock.*',
-    '!src/**/*.script.*',
-    '!**/__snapshots__',
-    '!**/__exclude',
-    'static',
-    'secret/**/*.enc', // encrypted secrets
-    'package.json',
-    'yarn.lock',
-    '.yarnrc',
-    'pnpm-lock.yaml',
-    'pnpm-workspace.yaml',
-    // 'tsconfig.json', // for path-mapping to work!
-    // 'tsconfig.dist.json',
-    '.gcloudignore',
-    'app.yaml',
-    'patches', // to allow patch-package
-    'resources',
-];
-const defaultFilesDir = `${srcDir}/deploy/files-default`;
-export async function deployPrepare(opt = {}) {
-    loadEnvFileIfExists();
-    const { projectDir = '.', targetDir = './tmp/deploy', createNpmrc = true } = opt;
-    const backendCfg = getBackendCfg(projectDir);
-    const inputPatterns = backendCfg.files || DEFAULT_FILES;
-    const appYamlPassEnv = opt.appYamlPassEnv || backendCfg.appYamlPassEnv;
-    console.log(`1. Copy files to ${dimGrey(targetDir)}`);
-    // Clean targetDir
-    fs2.emptyDir(targetDir);
-    kpySync({
-        baseDir: defaultFilesDir,
-        outputDir: targetDir,
-        dotfiles: true,
-    });
-    kpySync({
-        baseDir: projectDir,
-        inputPatterns,
-        outputDir: targetDir,
-        dotfiles: true,
-    });
-    const { NPM_TOKEN } = process.env;
-    if (NPM_TOKEN && createNpmrc) {
-        const npmrcPath = `${targetDir}/.npmrc`;
-        const npmrc = `//registry.npmjs.org/:_authToken=${NPM_TOKEN}`;
-        fs2.writeFile(npmrcPath, npmrc);
-    }
-    console.log(`2. Generate ${dimGrey('deployInfo.json')} and ${dimGrey('app.yaml')} in targetDir`);
-    const deployInfo = await createAndSaveDeployInfo(backendCfg, targetDir);
-    createAndSaveAppYaml(backendCfg, deployInfo, projectDir, targetDir, appYamlPassEnv);
-    return deployInfo;
-}

package/src/bin/deploy-gae.ts

@@ -1,14 +0,0 @@
-import { _parseArgs } from '@naturalcycles/nodejs-lib/args'
-import { runScript } from '@naturalcycles/nodejs-lib/runScript'
-import { deployGae } from '../deploy/deployGae.js'
-import { deployHealthCheckCliOptions } from '../deploy/deployHealthCheck.js'
-import { deployPrepareCliOptions } from '../deploy/deployPrepare.js'
-
-runScript(async () => {
-  const opt = _parseArgs({
-    ...deployPrepareCliOptions,
-    ...deployHealthCheckCliOptions,
-  })
-
-  await deployGae(opt)
-})

package/src/bin/deploy-prepare.ts

@@ -1,24 +0,0 @@
-/*
-
-yarn deploy-prepare
-
- */
-
-import { _parseArgs } from '@naturalcycles/nodejs-lib/args'
-import { runScript } from '@naturalcycles/nodejs-lib/runScript'
-import { deployPrepare, deployPrepareCliOptions } from '../deploy/deployPrepare.js'
-
-runScript(async () => {
-  const opt = _parseArgs(deployPrepareCliOptions)
-
-  await deployPrepare(opt)
-})
-
-// deploy strategy
-// gae project: from config
-// gae service: from config -- branch name
-// gae version: automatic form date
-
-// yarn build-prod
-// yarn deploy-prepare && ./tmp/deploy/app.yaml && ./tmp/deploy/deployInfo.json && json2env
-//

package/src/bin/undeploy-gae.ts

@@ -1,15 +0,0 @@
-import { _parseArgs } from '@naturalcycles/nodejs-lib/args'
-import { runScript } from '@naturalcycles/nodejs-lib/runScript'
-import { undeployGae } from '../deploy/deployGae.js'
-
-runScript(async () => {
-  const { branch } = _parseArgs({
-    branch: {
-      type: 'string',
-      demandOption: true,
-      desc: `Because Github Actions delete event happens after the branch is already deleted - you need to pass it manually`,
-    },
-  })
-
-  await undeployGae(branch)
-})

package/src/deploy/backend.cfg.util.ts

@@ -1,63 +0,0 @@
-import type { StringMap } from '@naturalcycles/js-lib/types'
-import { j } from '@naturalcycles/nodejs-lib/ajv'
-import { fs2 } from '@naturalcycles/nodejs-lib/fs2'
-import { yaml2 } from '@naturalcycles/nodejs-lib/yaml2'
-
-export interface BackendCfg {
-  gaeProject: string
-  gaeProjectByBranch?: StringMap
-
-  /**
-   * @example default
-   */
-  gaeService: string
-  gaeServiceByBranch?: StringMap
-
-  /**
-   * List of file patterns to include in deployment.
-   */
-  files?: string[]
-
-  appEnvDefault: string
-  appEnvByBranch?: StringMap
-
-  /**
-   * List of branches to use timestamps in gae version names (to keep previous versions).
-   */
-  branchesWithTimestampVersions?: string[]
-
-  /**
-   * If true - branch names are not passed into deployed urls as is, but are hashed.
-   */
-  hashedBranches?: boolean
-
-  /**
-   * Comma-separated list of env variables that will be passed to app.yaml from process.env
-   */
-  appYamlPassEnv?: string
-}
-
-export const backendCfgSchema = j.object<BackendCfg>({
-  gaeProject: j.string(),
-  gaeProjectByBranch: j.object.stringMap(j.string()).optional(),
-  gaeService: j.string(),
-  gaeServiceByBranch: j.object.stringMap(j.string()).optional(),
-  files: j.array(j.string()).optional(),
-  appEnvDefault: j.string(),
-  appEnvByBranch: j.object.stringMap(j.string()).optional(),
-  branchesWithTimestampVersions: j.array(j.string()).optional(),
-  hashedBranches: j.boolean().optional(),
-  appYamlPassEnv: j.string().optional(),
-})
-
-export function getBackendCfg(projectDir = '.'): BackendCfg {
-  const backendCfgYamlPath = `${projectDir}/backend.cfg.yaml`
-
-  fs2.requireFileToExist(backendCfgYamlPath)
-
-  const backendCfg: BackendCfg = {
-    ...yaml2.readYaml(backendCfgYamlPath),
-  }
-
-  return backendCfgSchema.validate(backendCfg)
-}

package/src/deploy/deploy.model.ts

@@ -1,32 +0,0 @@
-import type { UnixTimestamp } from '@naturalcycles/js-lib/types'
-
-export interface DeployInfo {
-  gaeProject: string
-  gaeService: string
-  gaeVersion: string
-  versionUrl: string
-  serviceUrl: string
-  gitRev: string
-  gitBranch: string
-
-  /**
-   * Unix timestamp of deployInfo.json being generated.
-   */
-  ts: UnixTimestamp
-}
-
-export interface AppYaml extends Record<string, any> {
-  runtime: string
-  service: string
-  inbound_services?: string[]
-  instance_class?: string
-  automatic_scaling?: {
-    min_instances?: number
-    max_instances?: number
-  }
-  env_variables: {
-    APP_ENV: string
-    DEBUG: string
-    [k: string]: string
-  }
-}