@modelprofile.com/flexharness 6.1.0 → 7.0.1

package/ts_document/readme.md

@@ -0,0 +1,49 @@
+PDF processing and language-model document recipes.
+
+Part of the [FlexHarness toolbox](https://code.foss.global/modelprofile.com/flexharness).
+Install the components your project uses. Provider adapters are registered explicitly;
+the agent and harness do not install or register vendor SDKs.
+
+```sh
+pnpm add @modelprofile.com/flexharness @push.rocks/smartpdf
+```
+
+```typescript
+import { extractTextFromPdf, stopSmartpdf } from '@modelprofile.com/flexharness/documents';
+
+try {
+  const text = await extractTextFromPdf(pdfBytes, { model });
+  console.log(text);
+} finally {
+  await stopSmartpdf();
+}
+```
+
+Supply a model from your selected FlexHarness provider. `extractTextFromPdf` replaces SmartOCR's `extractTextFromPdfBufferWithSmartAi`: it renders PDF pages through SmartPDF and asks the model to preserve text, page order, headings and table structure. Override `systemMessage` and `userMessage` or supply `messageHistory` when needed.
+
+`analyzeDocuments({ model, userMessage, pdfDocuments, systemMessage?, messageHistory? })` supports general analysis across multiple PDFs. This component owns a shared SmartPDF instance; call `stopSmartpdf()` when all document work has finished. Image-only OCR is available independently in `flexharness/ocr`.
+
+## Issue Reporting and Security
+
+Report issues and security concerns through [community.foss.global](https://community.foss.global).
+
+## License and Legal Information
+
+This repository contains open-source code licensed under the MIT License. A copy of the license can be found in the repository license file.
+
+**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.
+
+### Trademarks
+
+This project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH or third parties, and are not included within the scope of the MIT license granted herein.
+
+Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines or the guidelines of the respective third-party owners, and any usage must be approved in writing. Third-party trademarks used herein are the property of their respective owners and used only in a descriptive manner, e.g. for an implementation of an API or similar.
+
+### Company Information
+
+Task Venture Capital GmbH<br>
+Registered at District Court Bremen HRB 35230 HB, Germany
+
+For any legal inquiries or further information, please contact us via email at hello@task.vc.
+
+By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.

package/ts_kvm/classes.browserkvm.ts

@@ -0,0 +1,335 @@
+import * as plugins from './plugins.js';
+import type {
+  IBrowserKvmOptions,
+  IKvmDriver,
+  IKvmFrame,
+  IKvmTypeTextOptions,
+  TKvmKey,
+  TKvmKind,
+} from './interfaces.js';
+
+const defaultViewerSelector = 'video, canvas';
+
+interface ICaptureMediaResult {
+  captured: boolean;
+  width?: number;
+  height?: number;
+  dataBase64?: string;
+  error?: string;
+  hasMediaElement?: boolean;
+  mediaHasFrame?: boolean;
+}
+
+export class BrowserKvm implements IKvmDriver {
+  public readonly kind: TKvmKind;
+
+  private options: IBrowserKvmOptions;
+  private browser?: plugins.puppeteer.Browser;
+  private page?: plugins.puppeteer.Page;
+
+  constructor(options: IBrowserKvmOptions) {
+    this.options = options;
+    this.kind = options.kind ?? 'generic';
+  }
+
+  public async connect(): Promise<void> {
+    if (this.browser && this.page) {
+      return;
+    }
+
+    const timeoutMs = this.options.timeoutMs ?? 30000;
+    const args: string[] = [];
+    if (process.env.CI || process.getuid?.() === 0) {
+      args.push('--no-sandbox', '--disable-setuid-sandbox');
+    }
+
+    this.browser = await plugins.puppeteer.launch({
+      args,
+      acceptInsecureCerts: this.options.ignoreHttpsErrors ?? false,
+      defaultViewport: null,
+      executablePath: this.options.executablePath,
+      headless: this.options.headless ?? true,
+      timeout: timeoutMs,
+      userDataDir: this.options.userDataDir,
+    });
+
+    this.page = await this.browser.newPage();
+    this.page.setDefaultTimeout(timeoutMs);
+    this.page.setDefaultNavigationTimeout(timeoutMs);
+
+    await this.page.goto(this.options.url, {
+      waitUntil: 'domcontentloaded',
+      timeout: timeoutMs,
+    });
+
+    await this.tryGenericLogin();
+    await this.waitForViewerReady();
+    await this.focusViewer();
+  }
+
+  public async disconnect(): Promise<void> {
+    const browser = this.browser;
+    this.page = undefined;
+    this.browser = undefined;
+    if (browser) {
+      await browser.close();
+    }
+  }
+
+  public async focusViewer(): Promise<void> {
+    const page = this.requirePage();
+    const viewerSelector = this.getViewerSelector();
+    const viewerElement = await page.$(viewerSelector);
+    if (!viewerElement) {
+      throw new Error(`KVM viewer selector missing: ${viewerSelector}`);
+    }
+    await viewerElement.click();
+  }
+
+  public async captureFrame(): Promise<IKvmFrame> {
+    const page = this.requirePage();
+    const captureSelector = this.getCaptureSelector();
+    const captureElement = await page.$(captureSelector);
+    if (!captureElement) {
+      throw new Error(`KVM capture selector missing: ${captureSelector}`);
+    }
+
+    const mediaResult = await page.evaluate((selector: string): ICaptureMediaResult => {
+      const rootElement = document.querySelector(selector);
+      if (!rootElement) {
+        return {
+          captured: false,
+          error: `KVM capture selector missing: ${selector}`,
+        };
+      }
+
+      const mediaElement = (rootElement.matches('video, canvas')
+        ? rootElement
+        : rootElement.querySelector('video, canvas')) as HTMLVideoElement | HTMLCanvasElement | null;
+
+      if (!mediaElement) {
+        return {
+          captured: false,
+          hasMediaElement: false,
+        };
+      }
+
+      let width = 0;
+      let height = 0;
+      if (mediaElement instanceof HTMLVideoElement) {
+        width = mediaElement.videoWidth;
+        height = mediaElement.videoHeight;
+      } else if (mediaElement instanceof HTMLCanvasElement) {
+        width = mediaElement.width;
+        height = mediaElement.height;
+      }
+
+      if (width <= 0 || height <= 0) {
+        return {
+          captured: false,
+          hasMediaElement: true,
+          mediaHasFrame: false,
+        };
+      }
+
+      try {
+        const canvasElement = document.createElement('canvas');
+        canvasElement.width = width;
+        canvasElement.height = height;
+        const context = canvasElement.getContext('2d');
+        if (!context) {
+          return {
+            captured: false,
+            hasMediaElement: true,
+            mediaHasFrame: true,
+            error: 'Could not create canvas 2D context for KVM frame capture.',
+          };
+        }
+        context.drawImage(mediaElement, 0, 0, width, height);
+        const dataUrl = canvasElement.toDataURL('image/png');
+        return {
+          captured: true,
+          width,
+          height,
+          dataBase64: dataUrl.slice(dataUrl.indexOf(',') + 1),
+        };
+      } catch (error) {
+        return {
+          captured: false,
+          hasMediaElement: true,
+          mediaHasFrame: true,
+          error: error instanceof Error ? error.message : String(error),
+        };
+      }
+    }, captureSelector);
+
+    if (mediaResult.captured) {
+      return {
+        timestamp: Date.now(),
+        width: mediaResult.width ?? 0,
+        height: mediaResult.height ?? 0,
+        mimeType: 'image/png',
+        dataBase64: mediaResult.dataBase64 ?? '',
+      };
+    }
+
+    if (mediaResult.error?.startsWith('KVM capture selector missing')) {
+      throw new Error(mediaResult.error);
+    }
+
+    if (mediaResult.hasMediaElement && mediaResult.mediaHasFrame === false) {
+      throw new Error(`KVM media element has no frame: ${captureSelector}`);
+    }
+
+    const boundingBox = await captureElement.boundingBox();
+    if (!boundingBox || boundingBox.width <= 0 || boundingBox.height <= 0) {
+      throw new Error(`KVM capture selector has no visible frame: ${captureSelector}`);
+    }
+
+    const screenshot = await captureElement.screenshot({
+      type: 'png',
+    });
+
+    return {
+      timestamp: Date.now(),
+      width: Math.round(boundingBox.width),
+      height: Math.round(boundingBox.height),
+      mimeType: 'image/png',
+      dataBase64: Buffer.from(screenshot).toString('base64'),
+    };
+  }
+
+  public async typeText(text: string, options?: IKvmTypeTextOptions): Promise<void> {
+    const page = this.requirePage();
+    await this.focusViewer();
+    await page.keyboard.type(text, {
+      delay: options?.delayMs,
+    });
+  }
+
+  public async pressKey(key: TKvmKey): Promise<void> {
+    const page = this.requirePage();
+    await this.focusViewer();
+    await page.keyboard.press(key as plugins.puppeteer.KeyInput);
+  }
+
+  public async pressShortcut(keys: TKvmKey[]): Promise<void> {
+    const page = this.requirePage();
+    await this.focusViewer();
+
+    const pressedKeys: TKvmKey[] = [];
+    try {
+      for (const key of keys) {
+        await page.keyboard.down(key as plugins.puppeteer.KeyInput);
+        pressedKeys.push(key);
+      }
+    } finally {
+      for (const key of pressedKeys.reverse()) {
+        await page.keyboard.up(key as plugins.puppeteer.KeyInput);
+      }
+    }
+  }
+
+  public async wait(milliseconds: number): Promise<void> {
+    await new Promise<void>((resolve) => setTimeout(resolve, milliseconds));
+  }
+
+  private async tryGenericLogin(): Promise<void> {
+    if (!this.options.username || !this.options.password) {
+      return;
+    }
+
+    const page = this.requirePage();
+    const usernameElement = await this.findFirstElement([
+      'input[name="username"]',
+      'input[autocomplete="username"]',
+      'input[type="email"]',
+      'input[type="text"]',
+    ]);
+    const passwordElement = await this.findFirstElement([
+      'input[name="password"]',
+      'input[autocomplete="current-password"]',
+      'input[type="password"]',
+    ]);
+
+    if (!usernameElement || !passwordElement) {
+      return;
+    }
+
+    await usernameElement.click({ clickCount: 3 });
+    await usernameElement.type(this.options.username);
+    await passwordElement.click({ clickCount: 3 });
+    await passwordElement.type(this.options.password);
+    await passwordElement.press('Enter');
+
+    await Promise.race([
+      page
+        .waitForNavigation({
+          waitUntil: 'domcontentloaded',
+          timeout: Math.min(this.options.timeoutMs ?? 30000, 10000),
+        })
+        .catch(() => undefined),
+      this.wait(1000),
+    ]);
+  }
+
+  private async waitForViewerReady(): Promise<void> {
+    const page = this.requirePage();
+    const viewerSelector = this.getViewerSelector();
+    await page.waitForFunction(
+      (selector: string) => {
+        const rootElement = document.querySelector(selector);
+        if (!rootElement) {
+          return false;
+        }
+
+        const mediaElement = (rootElement.matches('video, canvas')
+          ? rootElement
+          : rootElement.querySelector('video, canvas')) as HTMLVideoElement | HTMLCanvasElement | null;
+
+        if (mediaElement instanceof HTMLVideoElement) {
+          return mediaElement.videoWidth > 0 && mediaElement.videoHeight > 0;
+        }
+
+        if (mediaElement instanceof HTMLCanvasElement) {
+          return mediaElement.width > 0 && mediaElement.height > 0;
+        }
+
+        const boundingRect = rootElement.getBoundingClientRect();
+        return boundingRect.width > 0 && boundingRect.height > 0;
+      },
+      {
+        timeout: this.options.timeoutMs ?? 30000,
+      },
+      viewerSelector
+    );
+  }
+
+  private async findFirstElement(
+    selectors: string[]
+  ): Promise<plugins.puppeteer.ElementHandle<Element> | null> {
+    const page = this.requirePage();
+    for (const selector of selectors) {
+      const element = await page.$(selector);
+      if (element) {
+        return element;
+      }
+    }
+    return null;
+  }
+
+  private getViewerSelector(): string {
+    return this.options.viewerSelector ?? defaultViewerSelector;
+  }
+
+  private getCaptureSelector(): string {
+    return this.options.captureSelector ?? this.options.viewerSelector ?? defaultViewerSelector;
+  }
+
+  private requirePage(): plugins.puppeteer.Page {
+    if (!this.page) {
+      throw new Error('BrowserKvm is not connected. Call connect() first.');
+    }
+    return this.page;
+  }
+}

package/ts_kvm/classes.kvmterminal.ts

@@ -0,0 +1,103 @@
+import { createWrappedKvmCommand, parseWrappedKvmCommandOutput } from './commandwrappers.js';
+import type {
+  IKvmTerminalCommandResult,
+  IKvmTerminalOptions,
+} from './interfaces.js';
+
+export class KvmTerminal {
+  private options: IKvmTerminalOptions;
+
+  constructor(options: IKvmTerminalOptions) {
+    this.options = options;
+  }
+
+  public async bootstrap(): Promise<void> {
+    const osHint = this.options.osHint ?? 'unknown';
+
+    switch (osHint) {
+      case 'windows':
+        await this.options.kvm.pressShortcut(['Meta', 'R']);
+        await this.options.kvm.wait(500);
+        await this.options.kvm.typeText('powershell -NoLogo');
+        await this.options.kvm.pressKey('Enter');
+        await this.options.kvm.wait(1500);
+        break;
+      case 'macos':
+        await this.options.kvm.pressShortcut(['Meta', 'Space']);
+        await this.options.kvm.wait(500);
+        await this.options.kvm.typeText('Terminal');
+        await this.options.kvm.pressKey('Enter');
+        await this.options.kvm.wait(1500);
+        break;
+      case 'linux':
+        await this.options.kvm.pressShortcut(['Control', 'Alt', 'T']);
+        await this.options.kvm.wait(1500);
+        break;
+      case 'unknown':
+      default:
+        break;
+    }
+  }
+
+  public async runCommand(command: string): Promise<IKvmTerminalCommandResult> {
+    const wrappedCommand = createWrappedKvmCommand(command, this.options.shellHint ?? 'unknown');
+    const commandTimeoutMs = this.options.commandTimeoutMs ?? 30000;
+    const ocrPollIntervalMs = this.options.ocrPollIntervalMs ?? 500;
+    const startedAt = Date.now();
+    let rawOcrText = '';
+    let attempts = 0;
+
+    await this.options.kvm.focusViewer();
+    await this.options.kvm.typeText(wrappedCommand.textToType);
+    await this.options.kvm.pressKey('Enter');
+
+    while (true) {
+      attempts++;
+      rawOcrText = await this.observeText();
+      const parsedResult = parseWrappedKvmCommandOutput({
+        commandId: wrappedCommand.commandId,
+        startMarker: wrappedCommand.startMarker,
+        endMarkerPrefix: wrappedCommand.endMarkerPrefix,
+        rawText: rawOcrText,
+      });
+
+      if (parsedResult.completed) {
+        return {
+          commandId: wrappedCommand.commandId,
+          command,
+          completed: true,
+          timedOut: false,
+          exitCode: parsedResult.exitCode,
+          combinedText: parsedResult.combinedText,
+          rawOcrText,
+        };
+      }
+
+      const reachedMaxAttempts =
+        typeof this.options.ocrMaxAttempts === 'number' && attempts >= this.options.ocrMaxAttempts;
+      const reachedTimeout = Date.now() - startedAt >= commandTimeoutMs;
+      if (reachedMaxAttempts || reachedTimeout) {
+        return {
+          commandId: wrappedCommand.commandId,
+          command,
+          completed: false,
+          timedOut: true,
+          combinedText: rawOcrText,
+          rawOcrText,
+        };
+      }
+
+      const remainingTimeMs = commandTimeoutMs - (Date.now() - startedAt);
+      await this.options.kvm.wait(Math.min(ocrPollIntervalMs, Math.max(remainingTimeMs, 0)));
+    }
+  }
+
+  public async observeText(): Promise<string> {
+    const frame = await this.options.kvm.captureFrame();
+    const result = await this.options.ocrEngine.recognize(frame, {
+      crop: this.options.ocrCrop,
+      language: 'eng',
+    });
+    return result.text;
+  }
+}

package/ts_kvm/commandwrappers.ts

@@ -0,0 +1,83 @@
+import * as plugins from './plugins.js';
+import type { IWrappedKvmCommand, TKvmShellHint } from './interfaces.js';
+
+export interface IParseWrappedKvmCommandOutputOptions {
+  commandId: string;
+  startMarker: string;
+  endMarkerPrefix: string;
+  rawText: string;
+}
+
+export interface IParseWrappedKvmCommandOutputResult {
+  completed: boolean;
+  exitCode?: number;
+  combinedText: string;
+}
+
+export const createWrappedKvmCommand = (
+  command: string,
+  shellHint: TKvmShellHint
+): IWrappedKvmCommand => {
+  const commandId = plugins.crypto.randomUUID().replace(/-/g, '');
+  const startMarker = `SMARTKVM_START_${commandId}`;
+  const endMarkerPrefix = `SMARTKVM_END_${commandId}_`;
+
+  let textToType: string;
+  switch (shellHint) {
+    case 'powershell':
+      textToType = `$__smartkvm_id = "${commandId}"; Write-Output "${startMarker}"; $__smartkvm_status = 0; try { ${command}; if ($null -ne $LASTEXITCODE) { $__smartkvm_status = $LASTEXITCODE } } catch { Write-Output $_; $__smartkvm_status = 1 }; Write-Output "${endMarkerPrefix}$__smartkvm_status"`;
+      break;
+    case 'cmd':
+      textToType = `echo ${startMarker} & ${command} & echo ${endMarkerPrefix}%ERRORLEVEL%`;
+      break;
+    case 'bash':
+    case 'zsh':
+    case 'sh':
+    case 'unknown':
+    default:
+      textToType = `printf '\n${startMarker}\n'; ${command}; __smartkvm_status=$?; printf '\n${endMarkerPrefix}%s\n' "$__smartkvm_status"`;
+      break;
+  }
+
+  return {
+    commandId,
+    shellHint,
+    command,
+    textToType,
+    startMarker,
+    endMarkerPrefix,
+  };
+};
+
+export const parseWrappedKvmCommandOutput = (
+  options: IParseWrappedKvmCommandOutputOptions
+): IParseWrappedKvmCommandOutputResult => {
+  const normalizedRawText = options.rawText.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
+  const startIndex = normalizedRawText.indexOf(options.startMarker);
+  if (startIndex === -1) {
+    return {
+      completed: false,
+      combinedText: normalizedRawText,
+    };
+  }
+
+  const contentStartIndex = startIndex + options.startMarker.length;
+  const endIndex = normalizedRawText.indexOf(options.endMarkerPrefix, contentStartIndex);
+  if (endIndex === -1) {
+    return {
+      completed: false,
+      combinedText: normalizedRawText,
+    };
+  }
+
+  const endMarkerValueStartIndex = endIndex + options.endMarkerPrefix.length;
+  const endMarkerLine = normalizedRawText.slice(endMarkerValueStartIndex).split('\n')[0].trim();
+  const exitCodeMatch = endMarkerLine.match(/^-?\d+/);
+  const exitCode = exitCodeMatch ? Number.parseInt(exitCodeMatch[0], 10) : undefined;
+
+  return {
+    completed: true,
+    exitCode,
+    combinedText: normalizedRawText.slice(contentStartIndex, endIndex).trim(),
+  };
+};

package/ts_kvm/index.ts

@@ -0,0 +1,6 @@
+export * from './interfaces.js';
+export * from './classes.browserkvm.js';
+export * from './classes.kvmterminal.js';
+export * from './commandwrappers.js';
+export * from './tools.js';
+export * from './ocr.js';