jitsu-cli 0.0.0

package/lib/cli/extension/index.js

@@ -0,0 +1,147 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.loadBuild = exports.getConfigJson = exports.getDistFile = exports.validateTsConfig = exports.extensionCommands = exports.help = void 0;
+const tslib_1 = require("tslib");
+const chalk_1 = tslib_1.__importDefault(require("chalk"));
+const chalk_code_highlight_1 = require("../../lib/chalk-code-highlight");
+const fs = tslib_1.__importStar(require("fs"));
+const indent_1 = require("../../lib/indent");
+const create_1 = require("./create");
+const build_1 = require("./build");
+const test_1 = require("./test");
+const validate_config_1 = require("./validate-config");
+const template_1 = require("./template");
+const json5_1 = tslib_1.__importDefault(require("json5"));
+const exec_1 = require("./exec");
+const readline = tslib_1.__importStar(require("readline"));
+const sandbox_1 = require("@jitsu/node-bridge/sandbox");
+global.fetch = require("cross-fetch");
+const usage = `
+  · ${chalk_1.default.bold("jitsu-cli extension create")}
+    
+    Create a project for jitsu plugin with placeholder implementation for all components.
+    
+  · ${chalk_1.default.bold("jitsu-cli extension build <directory>")}
+    
+    Build project located in <directory>. If <directory> is not provided current directory is used.
+   
+  · ${chalk_1.default.bold("jitsu-cli extension test  <directory>")}
+    
+    Run tests for project in <directory>/__test__. If <directory> is not provided current directory is used.
+   
+  · ${chalk_1.default.bold("jitsu-cli extension validate-config -f file or -j {json}")}
+   
+    Validates configuration json object. ${chalk_1.default.bold("Note:")} run \`jitsu-cli extension build\` beforehand 
+    
+  · ${chalk_1.default.bold("jitsu-cli extension help")}
+   
+    Show this help 
+`;
+exports.help = `
+
+${chalk_1.default.bold("DESCRIPTION")}
+
+  The command will build jitsu extension and bundle a single JS file that will
+  placed to a location defined in main parameter of package.json (if not set, the bundle
+  will be placed in dist/index.js
+
+  You can (and should!) call \`jitsu-cli extension build\` in package.json. Example: ${chalk_code_highlight_1.chalkCode.json((0, indent_1.align)(JSON.stringify((0, template_1.packageJsonTemplate)({
+    packageName: "test-package",
+    type: "destination",
+}), null, 2), { indent: 4, lnBefore: 2 }))}
+
+  The code should be located in ./src folder. src/index.js (or ts) should be present
+                               
+${chalk_1.default.bold("TYPESCRIPT")}
+
+  Typescript is supported out of the box. Just add tsconfig.json to the root of the project
+
+${chalk_1.default.bold("COMMANDS")} ${(0, indent_1.align)(usage, { indent: 2, lnBefore: 2 })}
+`;
+exports.extensionCommands = {
+    exec: {
+        exec: exec_1.execDestinationExtension,
+        description: "Execute destination extension on a test dataset",
+        help: "",
+    },
+    "exec-src": {
+        exec: exec_1.execSourceExtension,
+        description: "Builds and execute source connector extension with a test credentials and output data",
+        help: "",
+    },
+    test: {
+        exec: test_1.test,
+        description: "Execute test on extension",
+        help: "Tests should be located in ./__test__ folder and follow *.test.ts pattern",
+    },
+    build: {
+        exec: build_1.build,
+        description: "Builds Jitsu extension",
+        help: "",
+    },
+    create: {
+        exec: create_1.create,
+        description: "Creates an empty project",
+        help: "",
+    },
+    "validate-config": {
+        exec: validate_config_1.validateConfig,
+        description: `Verifies a configuration. ${chalk_1.default.bold("Note:")} run \`jitsu-cli extension build\` first!`,
+        help: "",
+    },
+};
+function validateTsConfig(tsConfigPath) {
+    let tsConfig;
+    try {
+        tsConfig = json5_1.default.parse(fs.readFileSync(tsConfigPath, "utf8"));
+    }
+    catch (e) {
+        throw new Error(`${chalk_1.default.bold(tsConfigPath)} - syntax error: ${e.message}`);
+    }
+    if (tsConfig?.compilerOptions?.module !== "ES2020") {
+        throw new Error(`${chalk_1.default.bold(tsConfigPath)} error: compilerOptions.module should be set to ES2020!`);
+    }
+}
+exports.validateTsConfig = validateTsConfig;
+function getDistFile(packageJson) {
+    return packageJson.main || "dist/index.js";
+}
+exports.getDistFile = getDistFile;
+async function getFirstLine(pathToFile) {
+    const readable = fs.createReadStream(pathToFile);
+    const reader = readline.createInterface({ input: readable });
+    const line = await new Promise(resolve => {
+        reader.on("line", line => {
+            reader.close();
+            resolve(line);
+        });
+    });
+    readable.close();
+    return line;
+}
+function getConfigJson(jsonOrFile) {
+    if (jsonOrFile.trim().startsWith("{") && jsonOrFile.trim().endsWith("}")) {
+        return json5_1.default.parse(jsonOrFile.trim());
+    }
+    else {
+        if (!fs.existsSync(jsonOrFile)) {
+            throw new Error(`File ${jsonOrFile} does not exist!`);
+        }
+        return json5_1.default.parse(fs.readFileSync(jsonOrFile, "utf8"));
+    }
+}
+exports.getConfigJson = getConfigJson;
+async function loadBuild(file) {
+    let formatDefinition = await getFirstLine(file);
+    if (formatDefinition.trim() === "//format=es" || formatDefinition.trim() === "//format=esm") {
+        return Promise.resolve().then(() => tslib_1.__importStar(require(file)));
+    }
+    else if (formatDefinition.trim() === "//format=cjs" || formatDefinition.trim() === "//format=commonjs") {
+        const vm = (0, sandbox_1.sandbox)({ file });
+        return vm.runFile(file);
+    }
+    else {
+        throw new Error(`Unsupported build format - ${formatDefinition}`);
+    }
+}
+exports.loadBuild = loadBuild;

package/lib/cli/extension/template.d.ts

@@ -0,0 +1,30 @@
+import { ProjectTemplate } from "../../lib/template";
+export declare type TemplateVars = {
+    license?: "MIT" | "Other";
+    packageName: string;
+    type: "destination" | "source" | "transform";
+    jitsuVersion?: string;
+};
+export declare const packageJsonTemplate: ({ packageName, type, jitsuVersion }: TemplateVars) => {
+    name: string;
+    version: string;
+    description: string;
+    main: string;
+    scripts: {
+        clean: string;
+        build: string;
+        test: string;
+        "validate-config": string;
+        execute: string;
+    };
+    devDependencies: {
+        "@jitsu/types": string;
+        "@jitsu/jlib": string;
+        "ts-jest": string;
+        "jitsu-cli": string;
+        tslib: string;
+        typescript: string;
+    };
+    dependencies: {};
+};
+export declare const extensionProjectTemplate: ProjectTemplate<TemplateVars>;

package/lib/cli/extension/template.js

@@ -0,0 +1,237 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.extensionProjectTemplate = exports.packageJsonTemplate = void 0;
+const version_1 = require("../../lib/version");
+const packageJsonTemplate = ({ packageName, type, jitsuVersion = undefined }) => ({
+    name: `${packageName}`,
+    version: "0.0.1",
+    description: `Jitsu ${type} - ${packageName}`,
+    main: `dist/${packageName}.js`,
+    scripts: {
+        clean: "rm -rf ./dist",
+        build: "jitsu-cli extension build",
+        test: "jitsu-cli extension test",
+        "validate-config": "jitsu-cli extension validate-config",
+        execute: `jitsu-cli extension ${type == "destination" ? "exec" : "exec-src"}`,
+    },
+    devDependencies: {
+        "@jitsu/types": `${jitsuVersion || "^" + version_1.jitsuCliVersion}`,
+        "@jitsu/jlib": `${jitsuVersion || "^" + version_1.jitsuCliVersion}`,
+        "ts-jest": "^27.0.7",
+        "jitsu-cli": `${jitsuVersion || "^" + version_1.jitsuCliVersion}`,
+        tslib: "^2.3.1",
+        typescript: "^4.5.2",
+    },
+    dependencies: {},
+});
+exports.packageJsonTemplate = packageJsonTemplate;
+let destinationTest = ({ type = "destination" }) => {
+    if (type !== "destination") {
+        return undefined;
+    }
+    return `
+    import { JitsuDestinationContext } from "@jitsu/types/extension"
+    import { testDestination } from "jitsu-cli/lib/tests"
+    import { destination } from "../src"
+
+    testDestination({
+      name: "basic",
+      context: {
+        destinationId: "test",
+        destinationType: "mydest",
+        config: {}
+      },
+      destination: destination,
+      event: {
+        event_type: 'test',
+        a: 1
+      },
+      expectedResult: {
+        method: "POST",
+        url: "https://test.com",
+        body: { a: 2 },
+      },
+    })
+`;
+};
+let destinationCode = () => {
+    return `
+    import { DestinationFunction, DestinationMessage, JitsuDestinationContext, ConfigValidator} from "@jitsu/types/extension";
+    import { DefaultJitsuEvent } from "@jitsu/types/event";
+    
+    export type DestinationConfig = {
+      exampleParam: string
+    }  
+    
+    export const validator: ConfigValidator<DestinationConfig> = async (config: DestinationConfig) => {
+      if (config.exampleParam !== 'valid-config') {
+        return \`Invalid config: exampleParam expected to be 'valid-config', but actual value is: \${config.exampleParam}\`;
+      }
+      return true;
+    }
+
+    export const destination: DestinationFunction = (event: DefaultJitsuEvent, dstContext: JitsuDestinationContext<DestinationConfig>) => {
+      return { url: "https://test.com", method: "POST", body: { a: (event.a || 0) + 1 } };
+    };
+`;
+};
+let sourceCode = () => {
+    return `
+    import { SourceCatalog, StateService, StreamReader, StreamSink, StreamConfiguration } from "@jitsu/types/sources";
+    import { ConfigValidationResult, ExtensionDescriptor } from "@jitsu/types/extension";
+    
+    export interface SourceConfig {
+      user_id: string
+      my_source_param: string;
+    }
+    
+    export interface StreamConfig {
+      my_stream_param: string;
+    }
+    
+    async function validator(config: SourceConfig): Promise<ConfigValidationResult> {
+      //TODO: Check that provided config data allows to connect to third party API 
+      console.log("validator is not yet implemented");
+      return true;
+    }
+    
+    const descriptor: ExtensionDescriptor<SourceConfig> = {
+      id: "my_source",
+      displayName: "Source Example",
+      description:
+        "Example source that produces row with run number and source/stream configuration.",
+      configurationParameters: [
+        {
+          displayName: "User ID",
+          id: "user_id",
+          type: "string",
+          required: true,
+          documentation: "User Id",
+        },
+        {
+          displayName: "Example Parameter",
+          id: "my_source_param",
+          required: true,
+          type: "string",
+          documentation: \`
+            <div>
+                Example Parameter
+            </div>
+            \`,
+        }
+      ],
+    };
+    
+    const sourceCatalog: SourceCatalog<SourceConfig, StreamConfig> = async config => {
+      return [
+        {
+          type: "my_source_runs",
+          supportedModes: ["incremental"],
+          params: [
+            {
+              id: "my_stream_param",
+              displayName: "Stream Parameter",
+              type: "string",
+              documentation: \`
+                <div>
+                  Stream Parameter example.
+                </div>
+              \`,
+              required: true,
+            },
+          ]
+        }
+      ];
+    };
+    
+    const streamReader: StreamReader<SourceConfig, StreamConfig> = async (
+      sourceConfig: SourceConfig,
+      streamType: string,
+      streamConfiguration: StreamConfiguration<StreamConfig>,
+      streamSink: StreamSink,
+      services: { state: StateService }
+    ) => {
+        //Example of saved state usage. Read previous run number:
+        let runNumber = services.state.get("run_number") || 0;
+        runNumber++ 
+        streamSink.log("INFO", "Run number: " + runNumber);
+        streamSink.addRecord({
+            $id: runNumber,
+            $recordTimestamp: new Date(),
+            type: streamType,
+            ...sourceConfig,
+            ...streamConfiguration.parameters
+        });
+        //Save last run number to state
+        services.state.set("run_number", runNumber);
+    };
+    
+    export { streamReader, sourceCatalog, descriptor, validator };
+`;
+};
+let transformCode = () => {
+    return `
+    import {Destination, DestinationMessage, JitsuDestinationContext} from "@jitsu/types/destination";
+    import {DefaultJitsuEvent} from "@jitsu/types/event";
+    
+    //duplicate events 
+    const transform: TransformationFunction = (event: DefaultJitsuEvent) => {
+      return [event, {...event, eventn_ctx_event_id: event.eventn_ctx_event_id + "_2"}] 
+    }
+
+    export default jitsuAdapter;
+`;
+};
+let descriptor = {};
+descriptor["destination"] = (vars) => `
+  import {ExtensionDescriptor} from "@jitsu/types/extension";
+  import {destination, validator, DestinationConfig} from "./destination";
+
+  const descriptor: ExtensionDescriptor = {
+    id: "${vars.packageName}",
+    displayName: "${vars.packageName}",
+    icon: "",
+    description: "Jitsu destination - ${vars.packageName} (generated by 'jitsu-cli extension create')",
+    configurationParameters: [
+      {id: "exampleParam", type: "string", required: true, displayName: "Example param", documentation: "Documentation"}
+    ],
+  };
+
+  export { descriptor, destination, validator };
+`;
+descriptor["transform"] = (vars) => `
+  import {DestinationAdapter, DestinationDescriptor} from "@jitsu/types/destination";
+  import jitsuAdapter from "./adapter";
+
+  const adapter: DestinationAdapter = jitsuAdapter
+
+  const descriptor: DestinationDescriptor = {
+    type: "${vars.packageName}",
+    displayName: "${vars.packageName}",
+    icon: "",
+    description: "Jitsu ${vars.type} - ${vars.packageName} (generated by 'jitsu-cli extension create)'",
+    configurationParameters: [
+      //put configuration here
+    ]
+  }
+
+  export {descriptor, adapter}
+`;
+exports.extensionProjectTemplate = {
+    "__test__/destination.test.ts": destinationTest,
+    "src/destination.ts": vars => vars.type == "destination" && destinationCode(),
+    "src/transform.ts": vars => vars.type == "transform" && transformCode(),
+    "src/index.ts": vars => (vars.type == "source" ? sourceCode() : descriptor[vars.type](vars)),
+    "package.json": exports.packageJsonTemplate,
+    "tsconfig.json": {
+        compilerOptions: {
+            module: "ES2020",
+            target: "ES2021",
+            moduleResolution: "Node",
+            esModuleInterop: true,
+            outDir: "./dist",
+            rootDir: "./",
+        },
+        include: ["./src", "__test__"],
+    },
+};

package/lib/cli/extension/test.d.ts

@@ -0,0 +1,2 @@
+import { CommandResult } from "../../lib/command/types";
+export declare function test(args: string[]): Promise<CommandResult>;

package/lib/cli/extension/test.js

@@ -0,0 +1,34 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.test = void 0;
+const tslib_1 = require("tslib");
+const path_1 = tslib_1.__importDefault(require("path"));
+const log_1 = tslib_1.__importDefault(require("../../lib/log"));
+const chalk_1 = tslib_1.__importDefault(require("chalk"));
+const fs_1 = tslib_1.__importDefault(require("fs"));
+const jest_cli_1 = require("jest-cli");
+const _1 = require("./");
+async function test(args) {
+    const directory = args?.[0] || "";
+    let projectBase = path_1.default.isAbsolute(directory) ? directory : path_1.default.resolve(process.cwd() + "/" + directory);
+    (0, log_1.default)().info("🛂 Running tests for " + chalk_1.default.bold(projectBase));
+    let tsConfigPath = path_1.default.resolve(projectBase, "tsconfig.json");
+    const typescriptEnabled = fs_1.default.existsSync(tsConfigPath);
+    if (typescriptEnabled) {
+        (0, log_1.default)().info(`ℹ️ Found ${chalk_1.default.bold("tsconfig.json")}, typescript will be enabled`);
+    }
+    else {
+        return {
+            success: false,
+            message: `${chalk_1.default.bold("tsconfig.json")} is not found in the root of the project. Pure JS extensions are not supported yet`,
+        };
+    }
+    (0, _1.validateTsConfig)(tsConfigPath);
+    const jestArgs = ["--passWithNoTests", "--projects", projectBase];
+    if (typescriptEnabled) {
+        jestArgs.push("--preset", "ts-jest");
+    }
+    await (0, jest_cli_1.run)(jestArgs);
+    return { success: true };
+}
+exports.test = test;

package/lib/cli/extension/validate-config.d.ts

@@ -0,0 +1,2 @@
+import { CommandResult } from "../../lib/command/types";
+export declare function validateConfig(args: string[]): Promise<CommandResult>;

package/lib/cli/extension/validate-config.js

@@ -0,0 +1,62 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.validateConfig = void 0;
+const tslib_1 = require("tslib");
+const commander_1 = tslib_1.__importDefault(require("commander"));
+const chalk_1 = tslib_1.__importDefault(require("chalk"));
+const path_1 = tslib_1.__importDefault(require("path"));
+const fs_1 = tslib_1.__importDefault(require("fs"));
+const json5_1 = tslib_1.__importDefault(require("json5"));
+const log_1 = tslib_1.__importDefault(require("../../lib/log"));
+const index_1 = require("./index");
+const validation_1 = require("../../lib/validation");
+async function validateConfig(args) {
+    const program = new commander_1.default.Command();
+    program.option("-d, --dir <project_dir>", "Project directory");
+    program.option("-c, --config <config_file_or_json>", "Configuration file path or inline extension configuration JSON");
+    program.argument("[project_dir]");
+    program.parse(["dummy", "dummy", ...args]);
+    const opts = program.opts();
+    if (!opts.config) {
+        return {
+            success: false,
+            message: "Please define config object -c json_file_path or -c '{json_object:}'",
+        };
+    }
+    let configObj;
+    try {
+        configObj = (0, index_1.getConfigJson)(opts.config);
+    }
+    catch (e) {
+        return {
+            success: false,
+            message: `Can't parse config JSON: '${opts.config}' ${e.message})`,
+        };
+    }
+    let projectDir = opts.dir || ".";
+    (0, log_1.default)().info("Project dir: " + projectDir);
+    let packageFile = path_1.default.resolve(projectDir, "package.json");
+    if (!fs_1.default.existsSync(packageFile)) {
+        return { success: false, message: `Can't find package file ${packageFile}` };
+    }
+    let packageObj = json5_1.default.parse(fs_1.default.readFileSync(packageFile, "utf8"));
+    let distFile = path_1.default.resolve(projectDir, (0, index_1.getDistFile)(packageObj));
+    (0, log_1.default)().info("Dist file: " + distFile);
+    if (!fs_1.default.existsSync(distFile)) {
+        return { success: false, message: `Can't find dist file (${distFile}). Forgot to run jitsu-cli extension build ?` };
+    }
+    (0, log_1.default)().info("🤔 Loading build from " + chalk_1.default.bold(distFile));
+    let build = await (0, index_1.loadBuild)(distFile);
+    (0, log_1.default)().info("👍 Module loaded!");
+    if (!build.validator) {
+        return { success: false, message: "Build doesn't export validator symbol" };
+    }
+    (0, log_1.default)().info("🤔 Validating configuration " + JSON.stringify(configObj));
+    let validationError = await (0, validation_1.validateConfiguration)(configObj, build.validator);
+    if (validationError) {
+        return { success: false, message: `❌ ${validationError}` };
+    }
+    (0, log_1.default)().info("✅ Config is valid. Hooray!");
+    return { success: true };
+}
+exports.validateConfig = validateConfig;

package/lib/index.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/index.js

@@ -0,0 +1,7 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const run_1 = require("./run");
+(async function () {
+    const args = [...process.argv.slice(2)];
+    process.exit(await (0, run_1.run)(args));
+})();

package/lib/lib/chalk-code-highlight.d.ts

@@ -0,0 +1,14 @@
+import { Grammar } from "prismjs";
+export declare type ColorScheme = Record<string, string | null>;
+export declare const defaultColorScheme: {
+    punctuation: string;
+    operator: string;
+    string: string;
+    keyword: string;
+    "function-variable": null;
+};
+export declare function chalkCode(code: string, lang: Grammar, colorScheme?: ColorScheme): string;
+export declare namespace chalkCode {
+    var typescript: (code: string | TemplateStringsArray, colorScheme?: ColorScheme) => string;
+    var json: (code: string | TemplateStringsArray, colorScheme?: ColorScheme) => string;
+}

package/lib/lib/chalk-code-highlight.js

@@ -0,0 +1,41 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.chalkCode = exports.defaultColorScheme = void 0;
+const tslib_1 = require("tslib");
+const prismjs_1 = tslib_1.__importDefault(require("prismjs"));
+const chalk_1 = tslib_1.__importDefault(require("chalk"));
+exports.defaultColorScheme = {
+    punctuation: "#999",
+    operator: "#9a6e3a",
+    string: "#9a6e3a",
+    keyword: "b#07a",
+    "function-variable": null,
+};
+function chalkString(expr, str) {
+    if (expr.startsWith("b")) {
+        return chalk_1.default.bold(chalkString(expr.substring(1), str));
+    }
+    else {
+        return chalk_1.default.hex(expr)(str);
+    }
+}
+function chalkCode(code, lang, colorScheme = exports.defaultColorScheme) {
+    return prismjs_1.default.tokenize(code, prismjs_1.default.languages.javascript)
+        .map(element => {
+        if (typeof element === "string") {
+            return element;
+        }
+        else {
+            let highlight = colorScheme[element.type];
+            return highlight ? chalkString(highlight, element.content.toString()) : `${element.content}`;
+        }
+    })
+        .join("");
+}
+exports.chalkCode = chalkCode;
+chalkCode.typescript = (code, colorScheme = exports.defaultColorScheme) => {
+    return chalkCode(typeof code === "string" ? code : code.join("\n"), prismjs_1.default.languages.typescript, colorScheme);
+};
+chalkCode.json = (code, colorScheme = exports.defaultColorScheme) => {
+    return chalkCode(typeof code === "string" ? code : code.join("\n"), prismjs_1.default.languages.json, colorScheme);
+};

package/lib/lib/command/index.d.ts

@@ -0,0 +1,3 @@
+import { Command, CommandRegistry, CommandResult, HelpOptions } from "./types";
+export declare const executeCommand: (commands: CommandRegistry, args: string[], helpOpts: HelpOptions) => Promise<CommandResult>;
+export declare function subcommands(commands: CommandRegistry, helpOpts: HelpOptions): Command;