@bitblit/ratchet-node-only 6.0.145-alpha → 6.0.147-alpha

package/src/third-party/common-crawl/model/common-crawl-fetch-options.ts

@@ -0,0 +1,12 @@
+export interface CommonCrawlFetchOptions {
+  url: string;
+  index?: string;
+  from?: string;
+  to?: string;
+  showNumPages?: boolean;
+  matchType?: 'exact' | 'prefix' | 'host' | 'domain';
+  limit?: number;
+  sort?: 'asc' | 'desc';
+  page?: number;
+  pageSize?: number;
+}

package/src/third-party/common-crawl/model/common-crawl-scan.ts

@@ -0,0 +1,11 @@
+import { WarcEntry } from './warc-entry.js';
+import { DomainIndexEntryRaw } from './domain-index-entry-raw.js';
+import { CommonCrawlFetchOptions } from './common-crawl-fetch-options.js';
+
+export interface CommonCrawlScan {
+  options: CommonCrawlFetchOptions;
+  //indexes: IndexEntryRaw[];
+  pageIndexes: DomainIndexEntryRaw[];
+  parsed: WarcEntry[];
+  errors: any[];
+}

package/src/third-party/common-crawl/model/domain-index-entry-raw.ts

@@ -0,0 +1,14 @@
+export interface DomainIndexEntryRaw {
+  urlkey: string;
+  timestamp: string; // number
+  url: string;
+  mime: string;
+  'mime-detected': string;
+  status: string; //number
+  digest: string;
+  length: string; //number,
+  offset: string; //number
+  filename: string;
+  languages: string;
+  encoding: string;
+}

package/src/third-party/common-crawl/model/index-entry-raw.ts

@@ -0,0 +1,8 @@
+export interface IndexEntryRaw {
+  id: string;
+  name: string;
+  timegate: string; // url
+  'cdx-api': string; // url
+  from: string; // timestamp (ISO)
+  to: string; // timestamp (ISO)
+}

package/src/third-party/common-crawl/model/warc-entry-raw.ts

@@ -0,0 +1,5 @@
+export interface WarcEntryRaw {
+  protocol: string;
+  headers: Record<string, string>;
+  content: Buffer;
+}

package/src/third-party/common-crawl/model/warc-entry.ts

@@ -0,0 +1,5 @@
+export interface WarcEntry {
+  protocol: string;
+  headers: Record<string, string>;
+  content: string;
+}

package/src/third-party/git/git-ratchet.spec.ts

@@ -0,0 +1,11 @@
+import { GitRatchet } from './git-ratchet.js';
+import { describe, expect, test } from 'vitest';
+import { Logger } from '@bitblit/ratchet-common/logger/logger';
+
+describe('#gitRatchet', function () {
+  test('should fetch last commit data', async () => {
+    const res: any = await GitRatchet.getLastCommitSwallowException();
+    expect(res).toBeTruthy();
+    Logger.info('Got: %j', res);
+  }, 10_000); // Ever so often this is slow... and it is really an integration test anyway
+});

package/src/third-party/git/git-ratchet.ts

@@ -0,0 +1,107 @@
+import process from 'child_process';
+import { EsmRatchet } from '@bitblit/ratchet-common/lang/esm-ratchet';
+import { Logger } from '@bitblit/ratchet-common/logger/logger';
+
+// Mainly ripped from https://raw.githubusercontent.com/seymen/git-last-commit/master/source/index.js
+// All credit due to https://github.com/seymen
+export class GitRatchet {
+  private static readonly SPLIT_CHARACTER: string = '<##>';
+  private static readonly PRETTY_FORMAT: string[] = ['%h', '%H', '%s', '%f', '%b', '%at', '%ct', '%an', '%ae', '%cn', '%ce', '%N', ''];
+
+  public static async executeCommand(command: string, options: any): Promise<string> {
+    let dst: string = EsmRatchet.fetchDirName(import.meta.url);
+
+    if (!!options && !!options.dst) {
+      dst = options.dst;
+    }
+
+    return new Promise<string>((res, rej) => {
+      process.exec(command, { cwd: dst }, (err, stdout, stderr) => {
+        if (stdout === '') {
+          rej('this does not look like a git repo');
+        }
+
+        if (stderr) {
+          rej(stderr);
+        }
+
+        res(stdout);
+      });
+    });
+  }
+
+  private static getCommandString(splitChar: string): string {
+    return (
+      'git log -1 --pretty=format:"' +
+      GitRatchet.PRETTY_FORMAT.join(splitChar) +
+      '"' +
+      ' && git rev-parse --abbrev-ref HEAD' +
+      ' && git tag --contains HEAD'
+    );
+  }
+
+  public static async getLastCommitSwallowException(options: any = {}): Promise<GitCommitData> {
+    let rval: GitCommitData = null;
+    try {
+      rval = await this.getLastCommit(options);
+    } catch (err) {
+      Logger.warn('Failed to fetch git data : %s', err, err);
+    }
+    return rval;
+  }
+
+  public static async getLastCommit(options: any = {}): Promise<GitCommitData> {
+    const command: string = GitRatchet.getCommandString(GitRatchet.SPLIT_CHARACTER);
+
+    const res: string = await GitRatchet.executeCommand(command, options);
+
+    const a: string[] = res.split(GitRatchet.SPLIT_CHARACTER);
+
+    // e.g. master\n or master\nv1.1\n or master\nv1.1\nv1.2\n
+    const branchAndTags: string[] = a[a.length - 1].split('\n').filter((n) => n);
+    const branch: string = branchAndTags[0];
+    const tags: string[] = branchAndTags.slice(1);
+
+    const rval: GitCommitData = {
+      shortHash: a[0],
+      hash: a[1],
+      subject: a[2],
+      sanitizedSubject: a[3],
+      body: a[4],
+      authoredOn: a[5],
+      committedOn: a[6],
+      author: {
+        name: a[7],
+        email: a[8],
+      },
+      committer: {
+        name: a[9],
+        email: a[10],
+      },
+      notes: a[11],
+      branch,
+      tags,
+    };
+    return rval;
+  }
+}
+
+export interface GitCommitData {
+  shortHash: string;
+  hash: string;
+  subject: string;
+  sanitizedSubject: string;
+  body: string;
+  authoredOn: string;
+  committedOn: string;
+  author: GitCommitDataPerson;
+  committer: GitCommitDataPerson;
+  notes: string;
+  branch: string;
+  tags: string[];
+}
+
+export interface GitCommitDataPerson {
+  name: string;
+  email: string;
+}

package/src/third-party/slack/publish-ci-release-to-slack.spec.ts

@@ -0,0 +1,28 @@
+import { PublishCiReleaseToSlack } from './publish-ci-release-to-slack.js';
+import { describe, expect, test } from 'vitest';
+import { Logger } from '@bitblit/ratchet-common/logger/logger';
+import { GlobalRatchet } from '@bitblit/ratchet-common/lang/global-ratchet';
+
+describe('#publishCircleCiReleaseToSlack', function () {
+  test.skip('should fail if not in a circle ci environment', async () => {
+    try {
+      const _result: string = await PublishCiReleaseToSlack.process('https://testslack.erigir.com');
+      this.bail();
+    } catch (err) {
+      Logger.debug('Caught expected error : %s', err);
+      // Expected, return ok
+    }
+  });
+
+  test.skip('should not fail if in a circle ci environment', async () => {
+    GlobalRatchet.setGlobalEnvVar('CIRCLE_BUILD_NUM', '1');
+    GlobalRatchet.setGlobalEnvVar('CIRCLE_BRANCH', 'B');
+    GlobalRatchet.setGlobalEnvVar('CIRCLE_TAG', 'T');
+    GlobalRatchet.setGlobalEnvVar('CIRCLE_SHA1', 'S');
+    GlobalRatchet.setGlobalEnvVar('CIRCLE_USERNAME', 'cweiss');
+    GlobalRatchet.setGlobalEnvVar('CIRCLE_PROJECT_REPONAME', 'tester');
+
+    const result: string = await PublishCiReleaseToSlack.process('slackUrlHere');
+    expect(result).toEqual('ok');
+  });
+});

package/src/third-party/slack/publish-ci-release-to-slack.ts

@@ -0,0 +1,84 @@
+import { DateTime } from 'luxon';
+import fetch from 'cross-fetch';
+import { GitCommitData, GitRatchet } from '../git/git-ratchet.js';
+import { CiRunInformation } from '../../ci/ci-run-information.js';
+import { CiRunInformationUtil } from '../../ci/ci-run-information-util.js';
+import { ErrorRatchet } from '@bitblit/ratchet-common/lang/error-ratchet';
+import { Logger } from '@bitblit/ratchet-common/logger/logger';
+import { StringRatchet } from '@bitblit/ratchet-common/lang/string-ratchet';
+
+export class PublishCiReleaseToSlack {
+  public static async process(slackHookUrl: string, timezone = 'America/Los_Angeles'): Promise<string> {
+    if (!slackHookUrl) {
+      throw new Error('slackHookUrl must be defined');
+    }
+
+    const github: CiRunInformation = CiRunInformationUtil.createDefaultGithubActionsRunInformation();
+    const circle: CiRunInformation = CiRunInformationUtil.createDefaultCircleCiRunInformation();
+    const testing: CiRunInformation = CiRunInformationUtil.createTestingCiRunInformation();
+
+    const buildNum: string = github.buildNumber || circle.buildNumber || testing.buildNumber;
+    const userName: string = github.userName || circle.userName || testing.userName;
+    const projectName: string = github.projectName || circle.projectName || testing.projectName;
+    const branch: string = github.branch || circle.branch || testing.branch || '';
+    const tag: string = github.tag || circle.tag || testing.tag || '';
+    const commitHash: string = github.commitHash || circle.commitHash || testing.commitHash || '';
+    const localTime: string = DateTime.local().setZone(timezone).toFormat('MMMM Do yyyy, h:mm:ss a z');
+    const gitData: GitCommitData = await GitRatchet.getLastCommitSwallowException();
+
+    if (!buildNum || !userName || !projectName) {
+      throw ErrorRatchet.fErr(
+        'Missing at least one of build number, username, or repo name environmental variables : Github : %j CircleCi: %j Testing: %j',
+        github,
+        circle,
+        testing,
+      );
+    }
+
+    let message: string = StringRatchet.format('%s performed release %s on %s at %s', userName, tag + ' ' + branch, projectName, localTime);
+    if (!!gitData && !!gitData.subject) {
+      message += '\n\n' + gitData.subject;
+    }
+
+    Logger.info(
+      'Sending slack notification "%s" with build %s, branch %s, tag %s, sha %s, time: %s, url: %s',
+      message,
+      buildNum,
+      branch,
+      tag,
+      commitHash,
+      localTime,
+      StringRatchet.obscure(slackHookUrl, 2),
+    );
+
+    const response: Response = await fetch(slackHookUrl, {
+      method: 'POST', // *GET, POST, PUT, DELETE, etc.
+      mode: 'cors', // no-cors, cors, *same-origin
+      cache: 'no-cache', // *default, no-cache, reload, force-cache, only-if-cached
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      redirect: 'follow', // manual, *follow, error
+      body: JSON.stringify({ text: message }), // body data type must match "Content-Type" header
+    });
+    const bodyOut: string = await response.text();
+
+    Logger.info('Slack returned : %s', bodyOut);
+    return bodyOut;
+  }
+
+  /**
+   And, in case you are running this command line...
+   TODO: should use switches to allow setting the various non-filename params
+   **/
+  public static async runFromCliArgs(args: string[]): Promise<string> {
+    Logger.info('Running PublishCiReleaseToSlack from command line arguments');
+    const hook: string = args?.length ? args[0] : null;
+    if (hook) {
+      return PublishCiReleaseToSlack.process(hook);
+    } else {
+      Logger.infoP('Usage : ratchet-publish-circle-ci-release-to-slack {hookUrl} ...');
+      return null;
+    }
+  }
+}