@bitblit/ratchet-node-only 6.0.145-alpha → 6.0.147-alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -3
- package/src/build/ratchet-node-only-info.ts +19 -0
- package/src/ci/apply-ci-env-variables-to-files.spec.ts +30 -0
- package/src/ci/apply-ci-env-variables-to-files.ts +98 -0
- package/src/ci/ci-run-information-util.ts +48 -0
- package/src/ci/ci-run-information.ts +9 -0
- package/src/cli/abstract-ratchet-cli-handler.ts +33 -0
- package/src/cli/cli-ratchet.ts +34 -0
- package/src/cli/ratchet-cli-handler.ts +24 -0
- package/src/csv/csv-ratchet.spec.ts +59 -0
- package/src/csv/csv-ratchet.ts +211 -0
- package/src/export-builder/export-map-builder-config.ts +10 -0
- package/src/export-builder/export-map-builder-target-config.ts +5 -0
- package/src/export-builder/export-map-builder.spec.ts +22 -0
- package/src/export-builder/export-map-builder.ts +157 -0
- package/src/files/files-to-static-class.spec.ts +26 -0
- package/src/files/files-to-static-class.ts +101 -0
- package/src/files/unique-file-rename.ts +80 -0
- package/src/http/local-file-server.ts +129 -0
- package/src/http/local-server-cert.ts +72 -0
- package/src/jwt/jwt-ratchet-config.ts +18 -0
- package/src/jwt/jwt-ratchet-like.ts +19 -0
- package/src/jwt/jwt-ratchet.spec.ts +85 -0
- package/src/jwt/jwt-ratchet.ts +204 -0
- package/src/stream/buffer-writable.ts +16 -0
- package/src/stream/multi-stream.ts +15 -0
- package/src/stream/node-stream-ratchet.spec.ts +19 -0
- package/src/stream/node-stream-ratchet.ts +70 -0
- package/src/stream/string-writable.spec.ts +16 -0
- package/src/stream/string-writable.ts +14 -0
- package/src/third-party/angular/angular-aot-rollup-plugin.ts +18 -0
- package/src/third-party/common-crawl/common-crawl-service.ts +220 -0
- package/src/third-party/common-crawl/model/common-crawl-fetch-options.ts +12 -0
- package/src/third-party/common-crawl/model/common-crawl-scan.ts +11 -0
- package/src/third-party/common-crawl/model/domain-index-entry-raw.ts +14 -0
- package/src/third-party/common-crawl/model/index-entry-raw.ts +8 -0
- package/src/third-party/common-crawl/model/warc-entry-raw.ts +5 -0
- package/src/third-party/common-crawl/model/warc-entry.ts +5 -0
- package/src/third-party/git/git-ratchet.spec.ts +11 -0
- package/src/third-party/git/git-ratchet.ts +107 -0
- package/src/third-party/slack/publish-ci-release-to-slack.spec.ts +28 -0
- package/src/third-party/slack/publish-ci-release-to-slack.ts +84 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bitblit/ratchet-node-only",
|
|
3
|
-
"version": "6.0.
|
|
3
|
+
"version": "6.0.147-alpha",
|
|
4
4
|
"description": "Ratchet tools for use on node-only",
|
|
5
5
|
"note-on-side-effects": "Technically the entries in 'bin' below might be side effects, but they are called explicitly",
|
|
6
6
|
"sideEffects": false,
|
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
},
|
|
10
10
|
"type": "module",
|
|
11
11
|
"files": [
|
|
12
|
+
"src/**",
|
|
12
13
|
"lib/**",
|
|
13
14
|
"bin/**"
|
|
14
15
|
],
|
|
@@ -58,7 +59,7 @@
|
|
|
58
59
|
"run-file-server": "node bin/cli.js run-file-server --https true"
|
|
59
60
|
},
|
|
60
61
|
"dependencies": {
|
|
61
|
-
"@bitblit/ratchet-common": "6.0.
|
|
62
|
+
"@bitblit/ratchet-common": "6.0.147-alpha"
|
|
62
63
|
},
|
|
63
64
|
"optionalDependencies": {
|
|
64
65
|
"cheerio": "1.1.2",
|
|
@@ -69,7 +70,7 @@
|
|
|
69
70
|
"warc": "1.0.1"
|
|
70
71
|
},
|
|
71
72
|
"peerDependencies": {
|
|
72
|
-
"@bitblit/ratchet-common": "6.0.
|
|
73
|
+
"@bitblit/ratchet-common": "6.0.147-alpha",
|
|
73
74
|
"cheerio": "^1.1.2",
|
|
74
75
|
"csv": "^6.4.1",
|
|
75
76
|
"jsonwebtoken": "^9.0.2",
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { BuildInformation } from '@bitblit/ratchet-common/build/build-information';
|
|
2
|
+
|
|
3
|
+
export class RatchetNodeOnlyInfo {
|
|
4
|
+
// Empty constructor prevents instantiation
|
|
5
|
+
// eslint-disable-next-line @typescript-eslint/no-empty-function
|
|
6
|
+
private constructor() {}
|
|
7
|
+
|
|
8
|
+
public static buildInformation(): BuildInformation {
|
|
9
|
+
const val: BuildInformation = {
|
|
10
|
+
version: 'LOCAL-SNAPSHOT',
|
|
11
|
+
hash: 'LOCAL-HASH',
|
|
12
|
+
branch: 'LOCAL-BRANCH',
|
|
13
|
+
tag: 'LOCAL-TAG',
|
|
14
|
+
timeBuiltISO: 'LOCAL-TIME-ISO',
|
|
15
|
+
notes: 'LOCAL-NOTES',
|
|
16
|
+
};
|
|
17
|
+
return val;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { ApplyCiEnvVariablesToFiles } from './apply-ci-env-variables-to-files.js';
|
|
2
|
+
import { CiRunInformationUtil } from './ci-run-information-util.js';
|
|
3
|
+
import { describe, expect, test } from 'vitest';
|
|
4
|
+
import { Logger } from '@bitblit/ratchet-common/logger/logger';
|
|
5
|
+
import { GlobalRatchet } from '@bitblit/ratchet-common/lang/global-ratchet';
|
|
6
|
+
|
|
7
|
+
describe('#applyCiEnvVariablesToFiles', function () {
|
|
8
|
+
test.skip('should fail if not in a ci environment', async () => {
|
|
9
|
+
try {
|
|
10
|
+
const _result: number = await ApplyCiEnvVariablesToFiles.process(
|
|
11
|
+
['test1.txt'],
|
|
12
|
+
CiRunInformationUtil.createDefaultCircleCiRunInformation(),
|
|
13
|
+
);
|
|
14
|
+
this.bail();
|
|
15
|
+
} catch (err) {
|
|
16
|
+
Logger.debug('Caught expected error : %s', err['message']);
|
|
17
|
+
// Expected, return ok
|
|
18
|
+
}
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
test('should not fail if in a ci environment', async () => {
|
|
22
|
+
GlobalRatchet.setGlobalEnvVar('CIRCLE_BUILD_NUM', '1');
|
|
23
|
+
GlobalRatchet.setGlobalEnvVar('CIRCLE_BRANCH', 'B');
|
|
24
|
+
GlobalRatchet.setGlobalEnvVar('CIRCLE_TAG', 'T');
|
|
25
|
+
GlobalRatchet.setGlobalEnvVar('CIRCLE_SHA1', 'S');
|
|
26
|
+
|
|
27
|
+
const result: number = await ApplyCiEnvVariablesToFiles.process([], CiRunInformationUtil.createDefaultCircleCiRunInformation());
|
|
28
|
+
expect(result).toEqual(0);
|
|
29
|
+
});
|
|
30
|
+
});
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import fs from 'fs';
|
|
2
|
+
import { CiRunInformation } from './ci-run-information.js';
|
|
3
|
+
import { CiRunInformationUtil } from './ci-run-information-util.js';
|
|
4
|
+
import { RequireRatchet } from '@bitblit/ratchet-common/lang/require-ratchet';
|
|
5
|
+
import { Logger } from '@bitblit/ratchet-common/logger/logger';
|
|
6
|
+
import { ErrorRatchet } from '@bitblit/ratchet-common/lang/error-ratchet';
|
|
7
|
+
import { StringRatchet } from '@bitblit/ratchet-common/lang/string-ratchet';
|
|
8
|
+
|
|
9
|
+
export class ApplyCiEnvVariablesToFiles {
|
|
10
|
+
public static async process(
|
|
11
|
+
fileNames: string[],
|
|
12
|
+
cfg: CiRunInformation,
|
|
13
|
+
buildFinder = 'LOCAL-SNAPSHOT',
|
|
14
|
+
branchFinder = 'LOCAL-BRANCH',
|
|
15
|
+
hashFinder = 'LOCAL-HASH',
|
|
16
|
+
tagFinder = 'LOCAL-TAG',
|
|
17
|
+
timeFinder = 'LOCAL-TIME',
|
|
18
|
+
): Promise<number> {
|
|
19
|
+
RequireRatchet.notNullOrUndefined(cfg, 'cfg');
|
|
20
|
+
RequireRatchet.notNullOrUndefined(cfg.buildNumber, 'cfg.buildNumber');
|
|
21
|
+
RequireRatchet.notNullOrUndefined(cfg.localTime, 'cfg.localTime');
|
|
22
|
+
if (!fileNames) {
|
|
23
|
+
throw new Error('fileNames must be defined');
|
|
24
|
+
}
|
|
25
|
+
if (fileNames.length === 0) {
|
|
26
|
+
Logger.warn('Warning - no files supplied to process');
|
|
27
|
+
}
|
|
28
|
+
if (!cfg.buildNumber) {
|
|
29
|
+
ErrorRatchet.throwFormattedErr('%s env var not set - apparently not in a CI environment', cfg.buildNumber);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
Logger.info('Processing files %j with run info %j', cfg);
|
|
33
|
+
|
|
34
|
+
let foundCount = 0;
|
|
35
|
+
fileNames.forEach((f) => {
|
|
36
|
+
if (!fs.existsSync(f)) {
|
|
37
|
+
Logger.error('Could not find file %s to process, continuing', f);
|
|
38
|
+
} else {
|
|
39
|
+
try {
|
|
40
|
+
let contents: string = fs.readFileSync(f).toString();
|
|
41
|
+
contents = contents.split(buildFinder).join(cfg.buildNumber);
|
|
42
|
+
contents = contents.split(branchFinder).join(cfg.branch || '');
|
|
43
|
+
contents = contents.split(hashFinder).join(cfg.commitHash || '');
|
|
44
|
+
contents = contents.split(tagFinder).join(cfg.tag || '');
|
|
45
|
+
contents = contents.split(timeFinder).join(cfg.localTime || '');
|
|
46
|
+
fs.writeFileSync(f, contents);
|
|
47
|
+
foundCount++;
|
|
48
|
+
} catch (err) {
|
|
49
|
+
Logger.error('Error processing %s , continuing: %s', f, err, err);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
return foundCount;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
public static extractFileNames(): string[] {
|
|
58
|
+
let rval: string[] = [];
|
|
59
|
+
if (process && process.argv && process.argv.length > 3) {
|
|
60
|
+
rval = process.argv.slice(3);
|
|
61
|
+
}
|
|
62
|
+
return rval;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
public static extractVariableConfig(inName: string): CiRunInformation {
|
|
66
|
+
let rval: CiRunInformation = null;
|
|
67
|
+
const name: string = StringRatchet.trimToEmpty(inName).toLowerCase();
|
|
68
|
+
switch (name) {
|
|
69
|
+
case 'circleci':
|
|
70
|
+
rval = CiRunInformationUtil.createDefaultCircleCiRunInformation();
|
|
71
|
+
break;
|
|
72
|
+
case 'github':
|
|
73
|
+
rval = CiRunInformationUtil.createDefaultGithubActionsRunInformation();
|
|
74
|
+
break;
|
|
75
|
+
case 'test':
|
|
76
|
+
rval = CiRunInformationUtil.createTestingCiRunInformation();
|
|
77
|
+
break;
|
|
78
|
+
default:
|
|
79
|
+
ErrorRatchet.throwFormattedErr('Unrecognized env var config type : %s', name);
|
|
80
|
+
}
|
|
81
|
+
Logger.info('Using variable config : %j', rval);
|
|
82
|
+
|
|
83
|
+
return rval;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
And, in case you are running this command line...
|
|
88
|
+
TODO: should use switches to allow setting the various non-filename params
|
|
89
|
+
**/
|
|
90
|
+
public static async runFromCliArgs(args: string[]): Promise<number> {
|
|
91
|
+
if (args.length > 1) {
|
|
92
|
+
return ApplyCiEnvVariablesToFiles.process(args.slice(1), ApplyCiEnvVariablesToFiles.extractVariableConfig(args[0]));
|
|
93
|
+
} else {
|
|
94
|
+
Logger.infoP('Usage : apply-ci-env-variables-to-files {file1} {file2} ...');
|
|
95
|
+
return -1;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { DateTime } from 'luxon';
|
|
2
|
+
import { CiRunInformation } from './ci-run-information.js';
|
|
3
|
+
import { GlobalRatchet } from '@bitblit/ratchet-common/lang/global-ratchet';
|
|
4
|
+
|
|
5
|
+
export class CiRunInformationUtil {
|
|
6
|
+
public static readonly DEFAULT_TIME_FORMAT: string = 'yyyy-MM-dd HH:mm:ss a z';
|
|
7
|
+
public static readonly DEFAULT_TIME_ZONE: string = 'America/Los_Angeles';
|
|
8
|
+
|
|
9
|
+
public static createTestingCiRunInformation(timezone = CiRunInformationUtil.DEFAULT_TIME_ZONE): CiRunInformation {
|
|
10
|
+
const now: string = new Date().toISOString();
|
|
11
|
+
const rval: CiRunInformation = {
|
|
12
|
+
buildNumber: 'Test_buildNumberVar_' + now,
|
|
13
|
+
localTime: DateTime.local().setZone(timezone).toFormat(CiRunInformationUtil.DEFAULT_TIME_FORMAT),
|
|
14
|
+
branch: 'Test_branchVar_' + now,
|
|
15
|
+
tag: 'Test_tagVar_' + now,
|
|
16
|
+
commitHash: 'Test_hashVar_' + now,
|
|
17
|
+
userName: 'Test_userNameVar_' + now,
|
|
18
|
+
projectName: 'Test_projectNameVar_' + now,
|
|
19
|
+
};
|
|
20
|
+
return rval;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
public static createDefaultCircleCiRunInformation(timezone = CiRunInformationUtil.DEFAULT_TIME_ZONE): CiRunInformation {
|
|
24
|
+
const rval: CiRunInformation = {
|
|
25
|
+
buildNumber: GlobalRatchet.fetchGlobalEnvVar('CIRCLE_BUILD_NUM'),
|
|
26
|
+
branch: GlobalRatchet.fetchGlobalEnvVar('CIRCLE_BRANCH'),
|
|
27
|
+
tag: GlobalRatchet.fetchGlobalEnvVar('CIRCLE_TAG'),
|
|
28
|
+
commitHash: GlobalRatchet.fetchGlobalEnvVar('CIRCLE_SHA1'),
|
|
29
|
+
localTime: DateTime.local().setZone(timezone).toFormat(CiRunInformationUtil.DEFAULT_TIME_FORMAT),
|
|
30
|
+
userName: GlobalRatchet.fetchGlobalEnvVar('CIRCLE_USERNAME'),
|
|
31
|
+
projectName: GlobalRatchet.fetchGlobalEnvVar('CIRCLE_PROJECT_REPONAME'),
|
|
32
|
+
};
|
|
33
|
+
return rval;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
public static createDefaultGithubActionsRunInformation(timezone = CiRunInformationUtil.DEFAULT_TIME_ZONE): CiRunInformation {
|
|
37
|
+
const rval: CiRunInformation = {
|
|
38
|
+
buildNumber: GlobalRatchet.fetchGlobalEnvVar('GITHUB_RUN_NUMBER'),
|
|
39
|
+
branch: GlobalRatchet.fetchGlobalEnvVar('GITHUB_REF_NAME'),
|
|
40
|
+
tag: GlobalRatchet.fetchGlobalEnvVar('GITHUB_REF_NAME'),
|
|
41
|
+
commitHash: GlobalRatchet.fetchGlobalEnvVar('GITHUB_SHA'),
|
|
42
|
+
localTime: DateTime.local().setZone(timezone).toFormat(CiRunInformationUtil.DEFAULT_TIME_FORMAT),
|
|
43
|
+
userName: GlobalRatchet.fetchGlobalEnvVar('GITHUB_ACTOR'),
|
|
44
|
+
projectName: GlobalRatchet.fetchGlobalEnvVar('GITHUB_REPOSITORY'),
|
|
45
|
+
};
|
|
46
|
+
return rval;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { CliRatchet } from './cli-ratchet.js';
|
|
2
|
+
import { BuildInformation } from '@bitblit/ratchet-common/build/build-information';
|
|
3
|
+
|
|
4
|
+
export abstract class AbstractRatchetCliHandler {
|
|
5
|
+
abstract fetchHandlerMap(): Record<string, any>;
|
|
6
|
+
abstract fetchVersionInfo(): BuildInformation;
|
|
7
|
+
|
|
8
|
+
public async findAndExecuteHandler(useSubstringMatch: boolean = false): Promise<void> {
|
|
9
|
+
let handler: any = null;
|
|
10
|
+
const versionArgs: string[] = CliRatchet.argsAfterCommand(['version'], useSubstringMatch);
|
|
11
|
+
if (versionArgs) {
|
|
12
|
+
console.log('Version : ' + JSON.stringify(this.fetchVersionInfo()));
|
|
13
|
+
console.log('Process: ' + JSON.stringify(process?.argv) + ' VersionArgs: ' + JSON.stringify(versionArgs));
|
|
14
|
+
} else {
|
|
15
|
+
const handlerMap: Record<string, any> = this.fetchHandlerMap();
|
|
16
|
+
const keys: string[] = Object.keys(handlerMap);
|
|
17
|
+
let remainArgs: string[] = null;
|
|
18
|
+
for (let i = 0; i < keys.length && !handler; i++) {
|
|
19
|
+
remainArgs = CliRatchet.argsAfterCommand([keys[i], keys[i] + '.js']);
|
|
20
|
+
if (remainArgs) {
|
|
21
|
+
handler = handlerMap[keys[i]];
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
if (handler) {
|
|
25
|
+
console.debug('Running command with args ' + JSON.stringify(remainArgs));
|
|
26
|
+
await handler(remainArgs);
|
|
27
|
+
} else {
|
|
28
|
+
console.log('Unrecognized command : ', process.argv);
|
|
29
|
+
console.log('Valid commands are : ', Object.keys(handlerMap));
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
export class CliRatchet {
|
|
2
|
+
public static isCalledFromCLI(filenames: string[], useSubstringMatch: boolean = false): boolean {
|
|
3
|
+
let rval: boolean = false;
|
|
4
|
+
for (let i = 0; filenames && i < filenames.length && !rval; i++) {
|
|
5
|
+
rval = CliRatchet.indexOfCommandArgument(filenames[i], useSubstringMatch) !== null;
|
|
6
|
+
}
|
|
7
|
+
return rval;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
public static argsAfterCommand(filenames: string[], useSubstringMatch: boolean = false): string[] {
|
|
11
|
+
let rval: string[] = null;
|
|
12
|
+
if (process?.argv?.length && filenames?.length) {
|
|
13
|
+
let idx: number = null;
|
|
14
|
+
for (let i = 0; i < filenames.length && idx === null; i++) {
|
|
15
|
+
idx = CliRatchet.indexOfCommandArgument(filenames[i], useSubstringMatch);
|
|
16
|
+
}
|
|
17
|
+
rval = idx !== null ? process.argv.slice(idx + 1, process.argv.length) : null;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
return rval;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
public static isCalledFromCLISingle(filename: string, useSubstringMatch: boolean = false): boolean {
|
|
24
|
+
return CliRatchet.isCalledFromCLI([filename], useSubstringMatch);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
public static indexOfCommandArgument(filename: string, useSubstringMatch: boolean = false): number {
|
|
28
|
+
const contFileName: boolean[] = process.argv.map(
|
|
29
|
+
(arg) => (!useSubstringMatch && arg === filename) || (useSubstringMatch && arg.indexOf(filename) !== -1),
|
|
30
|
+
);
|
|
31
|
+
const idx: number = contFileName.indexOf(true);
|
|
32
|
+
return idx === -1 ? null : idx;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { RatchetNodeOnlyInfo } from '../build/ratchet-node-only-info.js';
|
|
2
|
+
import { ApplyCiEnvVariablesToFiles } from '../ci/apply-ci-env-variables-to-files.js';
|
|
3
|
+
import { FilesToStaticClass } from '../files/files-to-static-class.js';
|
|
4
|
+
import { PublishCiReleaseToSlack } from '../third-party/slack/publish-ci-release-to-slack.js';
|
|
5
|
+
import { AbstractRatchetCliHandler } from './abstract-ratchet-cli-handler.js';
|
|
6
|
+
import { UniqueFileRename } from '../files/unique-file-rename.js';
|
|
7
|
+
import { BuildInformation } from '@bitblit/ratchet-common/build/build-information';
|
|
8
|
+
import { LocalFileServer } from '../http/local-file-server.js';
|
|
9
|
+
|
|
10
|
+
export class RatchetCliHandler extends AbstractRatchetCliHandler {
|
|
11
|
+
fetchHandlerMap(): Record<string, any> {
|
|
12
|
+
return {
|
|
13
|
+
'apply-ci-env-variables-to-files': ApplyCiEnvVariablesToFiles.runFromCliArgs,
|
|
14
|
+
'files-to-static-class': FilesToStaticClass.runFromCliArgs,
|
|
15
|
+
'publish-ci-release-to-slack': PublishCiReleaseToSlack.runFromCliArgs,
|
|
16
|
+
'unique-file-rename': UniqueFileRename.runFromCliArgs,
|
|
17
|
+
'run-file-server': LocalFileServer.runLocalFileServerFromCliArgs,
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
fetchVersionInfo(): BuildInformation {
|
|
22
|
+
return RatchetNodeOnlyInfo.buildInformation();
|
|
23
|
+
}
|
|
24
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { Subject } from 'rxjs';
|
|
2
|
+
import { CsvRatchet } from './csv-ratchet.js';
|
|
3
|
+
import { Logger } from '@bitblit/ratchet-common/logger/logger';
|
|
4
|
+
import { PromiseRatchet } from '@bitblit/ratchet-common/lang/promise-ratchet';
|
|
5
|
+
import { StringWritable } from '../stream/string-writable.js';
|
|
6
|
+
import { describe, expect, test } from 'vitest';
|
|
7
|
+
|
|
8
|
+
describe('#streamObjectsToCsv', function () {
|
|
9
|
+
test('should parse a string', async () => {
|
|
10
|
+
const testString: string = 'a,b\n1,2\n3,4\n';
|
|
11
|
+
const out: TestItem[] = await CsvRatchet.stringParse<TestItem>(testString, CsvRatchet.defaultParseFunction<TestItem>);
|
|
12
|
+
|
|
13
|
+
expect(out).toBeTruthy();
|
|
14
|
+
expect(out.length).toEqual(2);
|
|
15
|
+
// Need a real parse function to make this a number, and I am not testing that part here
|
|
16
|
+
expect(out[0].a).toEqual('1');
|
|
17
|
+
expect(out[1].a).toEqual('3');
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
test('should generate csv data', async () => {
|
|
21
|
+
const output: TestItem[] = [
|
|
22
|
+
{ a: 1, b: '2' },
|
|
23
|
+
{ a: 3, b: '4' },
|
|
24
|
+
];
|
|
25
|
+
const testString: string = await CsvRatchet.generateCsvData(output);
|
|
26
|
+
|
|
27
|
+
expect(testString).toBeTruthy();
|
|
28
|
+
expect(testString.length).toBeGreaterThan(10);
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
test('should stream objects to a csv', async () => {
|
|
32
|
+
// Logger.setLevel(LoggerLevelName.debug);
|
|
33
|
+
const sub: Subject<TestItem> = new Subject<TestItem>();
|
|
34
|
+
const out: StringWritable = new StringWritable();
|
|
35
|
+
|
|
36
|
+
const prom: Promise<number> = CsvRatchet.streamObjectsToCsv<TestItem>(sub, out); //, opts);
|
|
37
|
+
|
|
38
|
+
for (let i = 1; i < 6; i++) {
|
|
39
|
+
Logger.debug('Proc : %d', i);
|
|
40
|
+
sub.next({ a: i, b: 'test ' + i + ' ,,' });
|
|
41
|
+
await PromiseRatchet.wait(10);
|
|
42
|
+
}
|
|
43
|
+
sub.complete();
|
|
44
|
+
|
|
45
|
+
Logger.debug('Waiting on write');
|
|
46
|
+
|
|
47
|
+
const result: number = await prom;
|
|
48
|
+
Logger.debug('Write complete');
|
|
49
|
+
const val: string = out.value;
|
|
50
|
+
|
|
51
|
+
expect(result).toEqual(5);
|
|
52
|
+
Logger.debug('Have res : %d and val : \n%s', result, val);
|
|
53
|
+
});
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
export interface TestItem {
|
|
57
|
+
a: number;
|
|
58
|
+
b: string;
|
|
59
|
+
}
|
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
/*
|
|
2
|
+
Functions for working with csv data
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import fs, { ReadStream } from 'fs';
|
|
6
|
+
import { Options as ParseOptions, parse } from 'csv-parse';
|
|
7
|
+
import { Options, stringify } from 'csv-stringify';
|
|
8
|
+
import { Subject, Subscription } from 'rxjs';
|
|
9
|
+
import { Readable, Writable } from 'stream';
|
|
10
|
+
import { Logger } from '@bitblit/ratchet-common/logger/logger';
|
|
11
|
+
import { RequireRatchet } from '@bitblit/ratchet-common/lang/require-ratchet';
|
|
12
|
+
import { MapRatchet } from '@bitblit/ratchet-common/lang/map-ratchet';
|
|
13
|
+
|
|
14
|
+
export class CsvRatchet {
|
|
15
|
+
public static defaultParseOptions(): ParseOptions {
|
|
16
|
+
const rval: ParseOptions = {
|
|
17
|
+
delimiter: ',',
|
|
18
|
+
columns: true,
|
|
19
|
+
};
|
|
20
|
+
return rval;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
public static defaultStringifyOptions(): Options {
|
|
24
|
+
const rval: Options = {
|
|
25
|
+
header: true,
|
|
26
|
+
};
|
|
27
|
+
return rval;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
public static async stringParse<T>(
|
|
31
|
+
input: string,
|
|
32
|
+
pf: ParseFunction<T>,
|
|
33
|
+
opts: ParseOptions = CsvRatchet.defaultParseOptions(),
|
|
34
|
+
): Promise<T[]> {
|
|
35
|
+
return CsvRatchet.streamParse<T>(Readable.from(input), pf, opts);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
public static async streamParse<T>(
|
|
39
|
+
readStream: Readable,
|
|
40
|
+
pf: ParseFunction<T>,
|
|
41
|
+
opts: ParseOptions = CsvRatchet.defaultParseOptions(),
|
|
42
|
+
): Promise<T[]> {
|
|
43
|
+
return new Promise((res, rej) => {
|
|
44
|
+
const rval: T[] = [];
|
|
45
|
+
const p = parse(opts);
|
|
46
|
+
|
|
47
|
+
p.on('readable', () => {
|
|
48
|
+
let record: any = p.read();
|
|
49
|
+
while (record) {
|
|
50
|
+
const newVal: T = pf(record);
|
|
51
|
+
if (newVal) {
|
|
52
|
+
rval.push(newVal);
|
|
53
|
+
} else {
|
|
54
|
+
// Logger.debug('Stripping %j', record);
|
|
55
|
+
}
|
|
56
|
+
record = p.read();
|
|
57
|
+
}
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
p.on('error', (err) => {
|
|
61
|
+
rej(err);
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
p.on('end', () => {
|
|
65
|
+
res(rval);
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
readStream.pipe(p);
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
public static async fileParse<T>(filename: string, pf: ParseFunction<T>): Promise<T[]> {
|
|
73
|
+
const readStream: ReadStream = fs.createReadStream(filename);
|
|
74
|
+
return CsvRatchet.streamParse<T>(readStream, pf);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
public static async generateCsvData(objectsToConvert: any[], opts: Options = CsvRatchet.defaultStringifyOptions()): Promise<string> {
|
|
78
|
+
Logger.silly('Converting %d items into csv file', objectsToConvert.length);
|
|
79
|
+
const genProm: Promise<string> = new Promise<string>((res, rej) => {
|
|
80
|
+
stringify(objectsToConvert, opts, function (err, data) {
|
|
81
|
+
if (err) {
|
|
82
|
+
rej(err);
|
|
83
|
+
} else {
|
|
84
|
+
res(data);
|
|
85
|
+
}
|
|
86
|
+
});
|
|
87
|
+
});
|
|
88
|
+
return genProm;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
public static async generateComparison(file1: string, file2: string, keyField: string): Promise<ComparisonResults> {
|
|
92
|
+
RequireRatchet.notNullOrUndefined(file1, 'file1');
|
|
93
|
+
RequireRatchet.notNullOrUndefined(file2, 'file2');
|
|
94
|
+
RequireRatchet.notNullOrUndefined(keyField, 'keyField');
|
|
95
|
+
|
|
96
|
+
Logger.info('Created csv compare with files %s and %s keyed on %s', file1, file2, keyField);
|
|
97
|
+
//const file1Raw: string = fs.readFileSync(file1).toString();
|
|
98
|
+
let file1Parsed: any[] = await this.streamParse(fs.createReadStream(file1), CsvRatchet.defaultParseFunction);
|
|
99
|
+
file1Parsed = file1Parsed.map((m) => {
|
|
100
|
+
const next: any = {};
|
|
101
|
+
Object.keys(m).forEach((k) => {
|
|
102
|
+
next[k.trim()] = m[k];
|
|
103
|
+
});
|
|
104
|
+
return next;
|
|
105
|
+
});
|
|
106
|
+
const file1Mapped: Map<string, any> = MapRatchet.mapByUniqueProperty<any, string>(file1Parsed, keyField);
|
|
107
|
+
//const file2Raw: string = fs.readFileSync(file2).toString();
|
|
108
|
+
|
|
109
|
+
let file2Parsed: any[] = await this.streamParse(fs.createReadStream(file2), CsvRatchet.defaultParseFunction);
|
|
110
|
+
/*
|
|
111
|
+
let file2Parsed: any[] = parsesync(file2Raw, {
|
|
112
|
+
columns: true,
|
|
113
|
+
skip_empty_lines: true,
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
*/
|
|
117
|
+
file2Parsed = file2Parsed.map((m) => {
|
|
118
|
+
const next: any = {};
|
|
119
|
+
Object.keys(m).forEach((k) => {
|
|
120
|
+
next[k.trim()] = m[k];
|
|
121
|
+
});
|
|
122
|
+
return next;
|
|
123
|
+
});
|
|
124
|
+
const file2Mapped: Map<string, any> = MapRatchet.mapByUniqueProperty<any, string>(file2Parsed, keyField);
|
|
125
|
+
|
|
126
|
+
const f1Only: string[] = [];
|
|
127
|
+
const f2Only: string[] = [];
|
|
128
|
+
const both: string[] = [];
|
|
129
|
+
|
|
130
|
+
Array.from(file1Mapped.keys()).forEach((f1k) => {
|
|
131
|
+
if (file2Mapped.has(f1k)) {
|
|
132
|
+
both.push(f1k);
|
|
133
|
+
} else {
|
|
134
|
+
f1Only.push(f1k);
|
|
135
|
+
}
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
Array.from(file2Mapped.keys()).forEach((f1k) => {
|
|
139
|
+
if (!file1Mapped.has(f1k)) {
|
|
140
|
+
f2Only.push(f1k);
|
|
141
|
+
}
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
const rval: ComparisonResults = {
|
|
145
|
+
file1Data: file1Mapped,
|
|
146
|
+
file2Data: file2Mapped,
|
|
147
|
+
file1OnlyKeys: f1Only,
|
|
148
|
+
file2OnlyKeys: f2Only,
|
|
149
|
+
bothFilesKeys: both,
|
|
150
|
+
};
|
|
151
|
+
|
|
152
|
+
return rval;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
public static async streamObjectsToCsv<T>(srcSubject: Subject<T>, output: Writable, inOpts?: Options): Promise<number> {
|
|
156
|
+
RequireRatchet.notNullOrUndefined(srcSubject, 'srcSubject');
|
|
157
|
+
RequireRatchet.notNullOrUndefined(output, 'output');
|
|
158
|
+
const opts: Options = inOpts || CsvRatchet.defaultStringifyOptions();
|
|
159
|
+
|
|
160
|
+
Logger.silly('Running pipe to csv output : %j', opts);
|
|
161
|
+
let count: number = 0;
|
|
162
|
+
const genProm: Promise<number> = new Promise<number>((res, rej) => {
|
|
163
|
+
const stringifier = stringify(opts);
|
|
164
|
+
stringifier.on('error', (err) => {
|
|
165
|
+
if (sub) {
|
|
166
|
+
sub.unsubscribe();
|
|
167
|
+
}
|
|
168
|
+
rej(err);
|
|
169
|
+
});
|
|
170
|
+
stringifier.on('finish', () => {
|
|
171
|
+
if (sub) {
|
|
172
|
+
sub.unsubscribe(); // Cleanup
|
|
173
|
+
}
|
|
174
|
+
res(count);
|
|
175
|
+
});
|
|
176
|
+
stringifier.pipe(output);
|
|
177
|
+
|
|
178
|
+
const sub: Subscription = srcSubject.subscribe(
|
|
179
|
+
(next) => {
|
|
180
|
+
Logger.debug('Adding %j to csv', next);
|
|
181
|
+
count++;
|
|
182
|
+
stringifier.write(next);
|
|
183
|
+
},
|
|
184
|
+
(err) => {
|
|
185
|
+
Logger.error('Error generating : %s', err);
|
|
186
|
+
rej(err);
|
|
187
|
+
},
|
|
188
|
+
() => {
|
|
189
|
+
Logger.debug('Finished');
|
|
190
|
+
stringifier.end();
|
|
191
|
+
},
|
|
192
|
+
);
|
|
193
|
+
});
|
|
194
|
+
return genProm;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
public static defaultParseFunction<T>(row: any): T {
|
|
198
|
+
return row as T;
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
type ParseFunction<T> = (row: any) => T;
|
|
203
|
+
|
|
204
|
+
export interface ComparisonResults {
|
|
205
|
+
file1OnlyKeys: string[];
|
|
206
|
+
file2OnlyKeys: string[];
|
|
207
|
+
bothFilesKeys: string[];
|
|
208
|
+
|
|
209
|
+
file1Data: Map<string, any>;
|
|
210
|
+
file2Data: Map<string, any>;
|
|
211
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { ExportMapBuilderTargetConfig } from './export-map-builder-target-config.js';
|
|
2
|
+
|
|
3
|
+
export interface ExportMapBuilderConfig {
|
|
4
|
+
sourceRoot?: string; // Folder, by default the folder src in the same folder as targetPackageJsonFile
|
|
5
|
+
targets?: ExportMapBuilderTargetConfig[]; // Defaults to using folder lib, .js for imports, .d.ts for types
|
|
6
|
+
includes?: RegExp[]; // If file, added, if folder, recursively descended. Locations mapped from sourceRoot, defaults to .* if not set
|
|
7
|
+
excludes?: RegExp[]; // Any file/folder matching isn't processed. Defaults to empty
|
|
8
|
+
targetPackageJsonFile?: string; // defaults to package.json at root
|
|
9
|
+
dryRun?: boolean;
|
|
10
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import path from 'path';
|
|
2
|
+
//import { fileURLToPath, URL } from 'url';
|
|
3
|
+
//import { Logger } from '../../common/logger';
|
|
4
|
+
import { describe, expect, test } from 'vitest';
|
|
5
|
+
import { ExportMapBuilder } from './export-map-builder.js';
|
|
6
|
+
import { ExportMapBuilderConfig } from './export-map-builder-config.js';
|
|
7
|
+
import { EsmRatchet } from '@bitblit/ratchet-common/lang/esm-ratchet';
|
|
8
|
+
import { Logger } from '@bitblit/ratchet-common/logger/logger';
|
|
9
|
+
|
|
10
|
+
const testDirname: string = path.join(EsmRatchet.fetchDirName(import.meta.url), '../../../common');
|
|
11
|
+
|
|
12
|
+
describe('#exportMapBuilder', function () {
|
|
13
|
+
test('should build an export map', async () => {
|
|
14
|
+
const cfg: ExportMapBuilderConfig = {
|
|
15
|
+
targetPackageJsonFile: path.join(testDirname, 'package.json'),
|
|
16
|
+
dryRun: true,
|
|
17
|
+
};
|
|
18
|
+
const output: Record<string, any> = ExportMapBuilder.process(cfg);
|
|
19
|
+
expect(output).not.toBeNull;
|
|
20
|
+
Logger.info('Output: %j', output);
|
|
21
|
+
});
|
|
22
|
+
});
|