@transai/connector-runner-file 0.1.0 → 0.2.0

package/src/lib/processor.ts

@@ -1,186 +0,0 @@
-import path from 'path';
-
-import {
-  ConnectorSDKInterface,
-  FileHandleInterface,
-  FileInfo,
-  FilesSDKInterface,
-  IntervalHandler,
-} from '@transai/connector-runtime-sdk';
-// eslint-disable-next-line @nx/enforce-module-boundaries
-import { FileHandler, ParsedFile } from '@xip-online-data/file-handler';
-
-import { FileConfig, FileSelectorConfig } from './types';
-
-export class Processor implements IntervalHandler {
-  static readonly DEFAULT_INTERVAL_SECONDS = 60;
-
-  readonly #connectorSDK: ConnectorSDKInterface<FileConfig>;
-
-  readonly #fileSelector: FileSelectorConfig;
-
-  readonly #fileHandler: FilesSDKInterface;
-
-  readonly #destinationFileHandler?: FilesSDKInterface;
-
-  readonly #fileReader: FileHandler;
-
-  readonly #fileSelectionRegex: RegExp;
-
-  constructor(
-    connectorSDK: ConnectorSDKInterface<FileConfig>,
-    fileSelector: FileSelectorConfig,
-    fileHandler: FilesSDKInterface,
-  ) {
-    this.#connectorSDK = connectorSDK;
-    this.#fileSelector = fileSelector;
-    this.#fileHandler = fileHandler;
-    this.#fileReader = new FileHandler(fileSelector.delimiter);
-
-    const { selector } = this.#fileSelector;
-    // eslint-disable-next-line security/detect-non-literal-regexp
-    this.#fileSelectionRegex = new RegExp(
-      typeof selector !== 'string' ? selector.pattern : selector,
-      typeof selector !== 'string' && selector.flags ? selector.flags : 'i',
-    );
-
-    if (this.#fileSelector.destinationDsn) {
-      this.#destinationFileHandler = this.#connectorSDK.files(
-        this.#fileSelector.destinationDsn,
-      );
-    } else if (this.#fileSelector.action === 'move') {
-      throw new Error(
-        `Destination DSN must be provided for 'move' action in file selector: ${this.name}`,
-      );
-    }
-  }
-
-  get name(): string {
-    const { selector } = this.#fileSelector;
-    return `file-processor-${typeof selector === 'string' ? selector : selector.pattern}`;
-  }
-
-  async onRun(): Promise<void> {
-    const processedFiles = await this.#listDirectory();
-    this.#connectorSDK.telemetry.increment(
-      'files.processed.count',
-      processedFiles,
-    );
-  }
-
-  async #listDirectory(dirPath = '/'): Promise<number> {
-    const contents = await this.#fileHandler.list(dirPath);
-    const numberOfProcessedFilesList = await Promise.all(
-      contents.map(async (fileInfo): Promise<number> => {
-        if (fileInfo.type === 'FILE') {
-          const processed = await this.#processFile(fileInfo, dirPath);
-          return processed ? 1 : 0;
-        }
-
-        if (this.#fileSelector.recursive === true) {
-          return this.#listDirectory(path.join(dirPath, fileInfo.name));
-        }
-
-        return Promise.resolve(0);
-      }),
-    );
-
-    return numberOfProcessedFilesList.reduce((acc, curr) => acc + curr, 0);
-  }
-
-  async #processFile(fileInfo: FileInfo, dirPath: string): Promise<boolean> {
-    const filePath = path.join(dirPath, fileInfo.name);
-    if (!this.#fileSelectionRegex.test(fileInfo.name)) {
-      this.#connectorSDK.logger.verbose(
-        `Skipping file at path: ${filePath} as it does not match selector regex`,
-      );
-      return false;
-    }
-
-    this.#connectorSDK.logger.debug(`Processing file at path: ${filePath}`);
-    let fileContent: FileHandleInterface | undefined;
-    let parsedContent: ParsedFile | undefined;
-
-    try {
-      fileContent = await this.#fileHandler.read(filePath);
-
-      // Parse file to JSON using FileHandler based on file type
-      parsedContent = await this.#fileReader.handleBuffer(
-        filePath,
-        fileContent.get(),
-        this.#fileSelector.optionalHeaders ?? [],
-        this.#fileSelector.optionalSettings,
-      );
-
-      if (!parsedContent) {
-        this.#connectorSDK.logger.info(
-          `Failed to parse file at path: ${filePath}`,
-        );
-        fileContent.close();
-        return false;
-      }
-
-      this.#connectorSDK.logger.debug(
-        `Parsed file successfully at path: ${filePath}`,
-      );
-    } catch (error) {
-      this.#connectorSDK.logger.error(
-        `Error parsing file at path: ${filePath}, error: ${error}`,
-      );
-      fileContent?.close();
-      return false;
-    }
-
-    try {
-      const data = [
-        {
-          ...parsedContent,
-          _filename: FileHandler.getFileName(filePath) ?? fileInfo.name,
-        } as object,
-      ];
-      const metadata = {
-        keyField: '_filename',
-        collection: `${this.#connectorSDK.config.datasourceIdentifier ?? 'file'}_${this.#fileSelector.identifier ?? this.#fileSelectionRegex.source}`,
-      };
-
-      if (this.#fileSelector.type === 'metric') {
-        await this.#connectorSDK.sender.metricsLegacy(
-          data as Array<never>,
-          metadata,
-        );
-      } else {
-        await this.#connectorSDK.sender.documents(data, metadata);
-      }
-    } catch (error) {
-      this.#connectorSDK.logger.error(
-        `Error sending parsed file from path: ${filePath}, error: ${error}`,
-      );
-      fileContent?.close();
-      return false;
-    }
-
-    try {
-      // eslint-disable-next-line default-case
-      switch (this.#fileSelector.action) {
-        case 'move':
-          await this.#destinationFileHandler?.write(fileInfo.name, fileContent);
-          await this.#fileHandler.delete(filePath);
-          this.#connectorSDK.logger.debug(
-            `"Moved" file at path: ${filePath} to ${this.#destinationFileHandler?.pathAsDsn(fileInfo.name)} after processing`,
-          );
-          break;
-
-        case 'delete':
-          await this.#fileHandler.delete(filePath);
-          this.#connectorSDK.logger.debug(
-            `Deleted file at path: ${filePath} after processing`,
-          );
-          break;
-      }
-    } finally {
-      fileContent?.close();
-    }
-
-    return true;
-  }
-}

package/src/lib/types.ts

@@ -1,51 +0,0 @@
-import { BaseConnectorConfig } from '@xip-online-data/types';
-
-export interface FileSelectorConfig {
-  /**
-   * A regex pattern or string to select matching files. If the identifier is not
-   * provided, the selector string itself will be used as the identifier.
-   */
-  selector:
-    | {
-        pattern: string;
-        flags?: string;
-      }
-    | string;
-
-  /**
-   * The action to take on the file after processing. 'move' to move the file to
-   * a different DSN (requires destinationDsn), 'delete' to delete the file, or
-   * 'nothing' to leave the file as is (default).
-   */
-  action?: 'move' | 'delete' | 'nothing';
-
-  /**
-   * If the action is 'move', the destination DSN where the file should be moved.
-   */
-  destinationDsn?: string;
-
-  /**
-   * An optional identifier for the file selector, used for the collection.
-   */
-  identifier?: string;
-
-  intervalSeconds?: number;
-
-  /**
-   * Look for files recursively in subdirectories.
-   */
-  recursive?: boolean;
-
-  delimiter?: string;
-  optionalHeaders?: Array<string>;
-  optionalSettings?: {
-    [key: string]: unknown;
-  };
-
-  type?: 'metric' | 'document';
-}
-
-export interface FileConfig extends BaseConnectorConfig {
-  dsn: string;
-  fileSelectors?: Array<FileSelectorConfig>;
-}

package/tsconfig.json

@@ -1,22 +0,0 @@
-{
-  "extends": "../../tsconfig.base.json",
-  "compilerOptions": {
-    "module": "commonjs",
-    "forceConsistentCasingInFileNames": true,
-    "strict": true,
-    "noImplicitOverride": true,
-    "noImplicitReturns": true,
-    "noFallthroughCasesInSwitch": true,
-    "noPropertyAccessFromIndexSignature": true
-  },
-  "files": [],
-  "include": [],
-  "references": [
-    {
-      "path": "./tsconfig.lib.json"
-    },
-    {
-      "path": "./tsconfig.spec.json"
-    }
-  ]
-}

package/tsconfig.lib.json

@@ -1,10 +0,0 @@
-{
-  "extends": "./tsconfig.json",
-  "compilerOptions": {
-    "outDir": "../../dist/out-tsc",
-    "declaration": true,
-    "types": ["node"]
-  },
-  "include": ["src/**/*.ts"],
-  "exclude": ["jest.config.ts", "src/**/*.spec.ts", "src/**/*.test.ts"]
-}

package/tsconfig.spec.json

@@ -1,14 +0,0 @@
-{
-  "extends": "./tsconfig.json",
-  "compilerOptions": {
-    "outDir": "../../dist/out-tsc",
-    "module": "commonjs",
-    "types": ["jest", "node"]
-  },
-  "include": [
-    "jest.config.ts",
-    "src/**/*.test.ts",
-    "src/**/*.spec.ts",
-    "src/**/*.d.ts"
-  ]
-}