@modelprofile.com/flexharness-models 10.5.0 → 10.7.0

package/dist_ts_models/index.d.ts

@@ -3,5 +3,6 @@ export * from './classes.modelregistry.js';
 export * from './smartai.cache.js';
 export * from './smartai.limits.js';
 export * from './smartai.failures.js';
+export * from './smartai.textfiles.js';
 export { generateText, streamText, tool, jsonSchema } from './plugins.js';
 export type { ModelMessage, ToolSet, StreamTextResult } from './plugins.js';

package/dist_ts_models/index.js

@@ -3,5 +3,6 @@ export * from './classes.modelregistry.js';
 export * from './smartai.cache.js';
 export * from './smartai.limits.js';
 export * from './smartai.failures.js';
+export * from './smartai.textfiles.js';
 export { generateText, streamText, tool, jsonSchema } from './plugins.js';
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi90c19tb2RlbHMvaW5kZXgudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsY0FBYyxpQkFBaUIsQ0FBQztBQUNoQyxjQUFjLDRCQUE0QixDQUFDO0FBQzNDLGNBQWMsb0JBQW9CLENBQUM7QUFDbkMsY0FBYyxxQkFBcUIsQ0FBQztBQUNwQyxjQUFjLHVCQUF1QixDQUFDO0FBQ3RDLE9BQU8sRUFBRSxZQUFZLEVBQUUsVUFBVSxFQUFFLElBQUksRUFBRSxVQUFVLEVBQUUsTUFBTSxjQUFjLENBQUMifQ==
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi90c19tb2RlbHMvaW5kZXgudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsY0FBYyxpQkFBaUIsQ0FBQztBQUNoQyxjQUFjLDRCQUE0QixDQUFDO0FBQzNDLGNBQWMsb0JBQW9CLENBQUM7QUFDbkMsY0FBYyxxQkFBcUIsQ0FBQztBQUNwQyxjQUFjLHVCQUF1QixDQUFDO0FBQ3RDLGNBQWMsd0JBQXdCLENBQUM7QUFDdkMsT0FBTyxFQUFFLFlBQVksRUFBRSxVQUFVLEVBQUUsSUFBSSxFQUFFLFVBQVUsRUFBRSxNQUFNLGNBQWMsQ0FBQyJ9

package/dist_ts_models/smartai.textfiles.d.ts

@@ -0,0 +1,22 @@
+import type * as plugins from './plugins.js';
+/**
+ * Whether a media type names text: any `text/*` type, a JSON, XML, YAML, CSV, script or similar
+ * `application/*` type, a structured-syntax `+json`, `+xml` or `+yaml` type, or any type that
+ * states its `charset`. `text` alone, the top-level form a prompt may use, counts as text.
+ */
+export declare function isTextMediaType(mediaType: string): boolean;
+/** The text a model reads for one text file: its name and media type, then its content, fenced. */
+export declare function formatTextFileForModel(file: {
+    filename?: string;
+    mediaType: string;
+    text: string;
+}): string;
+/**
+ * Sends the text files of user messages as text parts, for providers that read only images and
+ * PDFs as files. A file part whose data is inline text, or whose media type names text (see
+ * `isTextMediaType`), becomes one text part that carries the file name, the media type and the
+ * content decoded in the stated charset, UTF-8 by default. Images, PDFs, other binary files, URLs
+ * and provider file references are passed on unchanged. A file that names text but does not decode
+ * fails the call before the request is sent.
+ */
+export declare function createTextFilePartMiddleware(): plugins.LanguageModelV4Middleware;

package/dist_ts_models/smartai.textfiles.js

@@ -0,0 +1,133 @@
+const textualApplicationTypes = new Set([
+    'application/csv',
+    'application/ecmascript',
+    'application/graphql',
+    'application/javascript',
+    'application/json',
+    'application/jsonl',
+    'application/ndjson',
+    'application/sql',
+    'application/toml',
+    'application/typescript',
+    'application/x-javascript',
+    'application/x-ndjson',
+    'application/x-sh',
+    'application/x-toml',
+    'application/x-typescript',
+    'application/x-yaml',
+    'application/xml',
+    'application/yaml',
+]);
+const textualSuffixes = ['+json', '+xml', '+yaml'];
+function parseMediaType(mediaType) {
+    const [essence = '', ...parameters] = mediaType.split(';');
+    let charset;
+    for (const parameter of parameters) {
+        const separator = parameter.indexOf('=');
+        if (separator === -1)
+            continue;
+        if (parameter.slice(0, separator).trim().toLowerCase() !== 'charset')
+            continue;
+        charset = parameter.slice(separator + 1).trim().replace(/^"(.*)"$/u, '$1');
+    }
+    return { essence: essence.trim().toLowerCase(), ...(charset ? { charset } : {}) };
+}
+/**
+ * Whether a media type names text: any `text/*` type, a JSON, XML, YAML, CSV, script or similar
+ * `application/*` type, a structured-syntax `+json`, `+xml` or `+yaml` type, or any type that
+ * states its `charset`. `text` alone, the top-level form a prompt may use, counts as text.
+ */
+export function isTextMediaType(mediaType) {
+    const { essence, charset } = parseMediaType(mediaType);
+    if (charset !== undefined)
+        return true;
+    if (essence === 'text' || essence.startsWith('text/'))
+        return true;
+    if (textualApplicationTypes.has(essence))
+        return true;
+    return essence.startsWith('application/') && textualSuffixes.some((suffix) => essence.endsWith(suffix));
+}
+function decodeBase64(data) {
+    const binary = atob(data);
+    const bytes = new Uint8Array(binary.length);
+    for (let index = 0; index < binary.length; index++)
+        bytes[index] = binary.charCodeAt(index);
+    return bytes;
+}
+function describeFile(part) {
+    return part.filename === undefined ? `file part of media type ${part.mediaType}` : `file "${part.filename}"`;
+}
+function decodeText(part, data) {
+    const charset = parseMediaType(part.mediaType).charset ?? 'utf-8';
+    let decoder;
+    try {
+        decoder = new TextDecoder(charset, { fatal: true });
+    }
+    catch (error) {
+        throw new Error(`The ${describeFile(part)} names charset "${charset}", which cannot be decoded.`, { cause: error });
+    }
+    try {
+        return decoder.decode(typeof data === 'string' ? decodeBase64(data) : data);
+    }
+    catch (error) {
+        throw new Error(`The ${describeFile(part)} is not valid ${charset} text.`, { cause: error });
+    }
+}
+/** The text a model reads for one text file: its name and media type, then its content, fenced. */
+export function formatTextFileForModel(file) {
+    const longestBacktickRun = Math.max(0, ...Array.from(file.text.matchAll(/`+/gu), (match) => match[0].length));
+    const fence = '`'.repeat(Math.max(3, longestBacktickRun + 1));
+    const heading = file.filename === undefined
+        ? `Attached file (${file.mediaType}):`
+        : `Attached file ${file.filename} (${file.mediaType}):`;
+    const body = file.text.endsWith('\n') ? file.text : `${file.text}\n`;
+    return `${heading}\n${fence}\n${body}${fence}`;
+}
+function inlineTextFile(part) {
+    if (part.type !== 'file')
+        return part;
+    let text;
+    if (part.data.type === 'text') {
+        text = part.data.text;
+    }
+    else if (part.data.type === 'data' && isTextMediaType(part.mediaType)) {
+        text = decodeText(part, part.data.data);
+    }
+    else {
+        return part;
+    }
+    return {
+        type: 'text',
+        text: formatTextFileForModel({
+            ...(part.filename === undefined ? {} : { filename: part.filename }),
+            mediaType: part.mediaType,
+            text,
+        }),
+        ...(part.providerOptions === undefined ? {} : { providerOptions: part.providerOptions }),
+    };
+}
+/**
+ * Sends the text files of user messages as text parts, for providers that read only images and
+ * PDFs as files. A file part whose data is inline text, or whose media type names text (see
+ * `isTextMediaType`), becomes one text part that carries the file name, the media type and the
+ * content decoded in the stated charset, UTF-8 by default. Images, PDFs, other binary files, URLs
+ * and provider file references are passed on unchanged. A file that names text but does not decode
+ * fails the call before the request is sent.
+ */
+export function createTextFilePartMiddleware() {
+    return {
+        specificationVersion: 'v4',
+        transformParams: async ({ params }) => {
+            const hasTextFile = params.prompt.some((message) => message.role === 'user' && message.content.some((part) => part.type === 'file' && (part.data.type === 'text' || (part.data.type === 'data' && isTextMediaType(part.mediaType)))));
+            if (!hasTextFile)
+                return params;
+            return {
+                ...params,
+                prompt: params.prompt.map((message) => message.role === 'user'
+                    ? { ...message, content: message.content.map(inlineTextFile) }
+                    : message),
+            };
+        },
+    };
+}
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic21hcnRhaS50ZXh0ZmlsZXMuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi90c19tb2RlbHMvc21hcnRhaS50ZXh0ZmlsZXMudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBV0EsTUFBTSx1QkFBdUIsR0FBRyxJQUFJLEdBQUcsQ0FBQztJQUN0QyxpQkFBaUI7SUFDakIsd0JBQXdCO0lBQ3hCLHFCQUFxQjtJQUNyQix3QkFBd0I7SUFDeEIsa0JBQWtCO0lBQ2xCLG1CQUFtQjtJQUNuQixvQkFBb0I7SUFDcEIsaUJBQWlCO0lBQ2pCLGtCQUFrQjtJQUNsQix3QkFBd0I7SUFDeEIsMEJBQTBCO0lBQzFCLHNCQUFzQjtJQUN0QixrQkFBa0I7SUFDbEIsb0JBQW9CO0lBQ3BCLDBCQUEwQjtJQUMxQixvQkFBb0I7SUFDcEIsaUJBQWlCO0lBQ2pCLGtCQUFrQjtDQUNuQixDQUFDLENBQUM7QUFFSCxNQUFNLGVBQWUsR0FBRyxDQUFDLE9BQU8sRUFBRSxNQUFNLEVBQUUsT0FBTyxDQUFDLENBQUM7QUFFbkQsU0FBUyxjQUFjLENBQUMsU0FBaUI7SUFDdkMsTUFBTSxDQUFDLE9BQU8sR0FBRyxFQUFFLEVBQUUsR0FBRyxVQUFVLENBQUMsR0FBRyxTQUFTLENBQUMsS0FBSyxDQUFDLEdBQUcsQ0FBQyxDQUFDO0lBQzNELElBQUksT0FBMkIsQ0FBQztJQUNoQyxLQUFLLE1BQU0sU0FBUyxJQUFJLFVBQVUsRUFBRSxDQUFDO1FBQ25DLE1BQU0sU0FBUyxHQUFHLFNBQVMsQ0FBQyxPQUFPLENBQUMsR0FBRyxDQUFDLENBQUM7UUFDekMsSUFBSSxTQUFTLEtBQUssQ0FBQyxDQUFDO1lBQUUsU0FBUztRQUMvQixJQUFJLFNBQVMsQ0FBQyxLQUFLLENBQUMsQ0FBQyxFQUFFLFNBQVMsQ0FBQyxDQUFDLElBQUksRUFBRSxDQUFDLFdBQVcsRUFBRSxLQUFLLFNBQVM7WUFBRSxTQUFTO1FBQy9FLE9BQU8sR0FBRyxTQUFTLENBQUMsS0FBSyxDQUFDLFNBQVMsR0FBRyxDQUFDLENBQUMsQ0FBQyxJQUFJLEVBQUUsQ0FBQyxPQUFPLENBQUMsV0FBVyxFQUFFLElBQUksQ0FBQyxDQUFDO0lBQzdFLENBQUM7SUFDRCxPQUFPLEVBQUUsT0FBTyxFQUFFLE9BQU8sQ0FBQyxJQUFJLEVBQUUsQ0FBQyxXQUFXLEVBQUUsRUFBRSxHQUFHLENBQUMsT0FBTyxDQUFDLENBQUMsQ0FBQyxFQUFFLE9BQU8sRUFBRSxDQUFDLENBQUMsQ0FBQyxFQUFFLENBQUMsRUFBRSxDQUFDO0FBQ3BGLENBQUM7QUFFRDs7OztHQUlHO0FBQ0gsTUFBTSxVQUFVLGVBQWUsQ0FBQyxTQUFpQjtJQUMvQyxNQUFNLEVBQUUsT0FBTyxFQUFFLE9BQU8sRUFBRSxHQUFHLGNBQWMsQ0FBQyxTQUFTLENBQUMsQ0FBQztJQUN2RCxJQUFJLE9BQU8sS0FBSyxTQUFTO1FBQUUsT0FBTyxJQUFJLENBQUM7SUFDdkMsSUFBSSxPQUFPLEtBQUssTUFBTSxJQUFJLE9BQU8sQ0FBQyxVQUFVLENBQUMsT0FBTyxDQUFDO1FBQUUsT0FBTyxJQUFJLENBQUM7SUFDbkUsSUFBSSx1QkFBdUIsQ0FBQyxHQUFHLENBQUMsT0FBTyxDQUFDO1FBQUUsT0FBTyxJQUFJLENBQUM7SUFDdEQsT0FBTyxPQUFPLENBQUMsVUFBVSxDQUFDLGNBQWMsQ0FBQyxJQUFJLGVBQWUsQ0FBQyxJQUFJLENBQUMsQ0FBQyxNQUFNLEVBQUUsRUFBRSxDQUFDLE9BQU8sQ0FBQyxRQUFRLENBQUMsTUFBTSxDQUFDLENBQUMsQ0FBQztBQUMxRyxDQUFDO0FBRUQsU0FBUyxZQUFZLENBQUMsSUFBWTtJQUNoQyxNQUFNLE1BQU0sR0FBRyxJQUFJLENBQUMsSUFBSSxDQUFDLENBQUM7SUFDMUIsTUFBTSxLQUFLLEdBQUcsSUFBSSxVQUFVLENBQUMsTUFBTSxDQUFDLE1BQU0sQ0FBQyxDQUFDO0lBQzVDLEtBQUssSUFBSSxLQUFLLEdBQUcsQ0FBQyxFQUFFLEtBQUssR0FBRyxNQUFNLENBQUMsTUFBTSxFQUFFLEtBQUssRUFBRTtRQUFFLEtBQUssQ0FBQyxLQUFLLENBQUMsR0FBRyxNQUFNLENBQUMsVUFBVSxDQUFDLEtBQUssQ0FBQyxDQUFDO0lBQzVGLE9BQU8sS0FBSyxDQUFDO0FBQ2YsQ0FBQztBQUVELFNBQVMsWUFBWSxDQUFDLElBQWU7SUFDbkMsT0FBTyxJQUFJLENBQUMsUUFBUSxLQUFLLFNBQVMsQ0FBQyxDQUFDLENBQUMsMkJBQTJCLElBQUksQ0FBQyxTQUFTLEVBQUUsQ0FBQyxDQUFDLENBQUMsU0FBUyxJQUFJLENBQUMsUUFBUSxHQUFHLENBQUM7QUFDL0csQ0FBQztBQUVELFNBQVMsVUFBVSxDQUFDLElBQWUsRUFBRSxJQUF5QjtJQUM1RCxNQUFNLE9BQU8sR0FBRyxjQUFjLENBQUMsSUFBSSxDQUFDLFNBQVMsQ0FBQyxDQUFDLE9BQU8sSUFBSSxPQUFPLENBQUM7SUFDbEUsSUFBSSxPQUFvQixDQUFDO0lBQ3pCLElBQUksQ0FBQztRQUNILE9BQU8sR0FBRyxJQUFJLFdBQVcsQ0FBQyxPQUFPLEVBQUUsRUFBRSxLQUFLLEVBQUUsSUFBSSxFQUFFLENBQUMsQ0FBQztJQUN0RCxDQUFDO0lBQUMsT0FBTyxLQUFLLEVBQUUsQ0FBQztRQUNmLE1BQU0sSUFBSSxLQUFLLENBQUMsT0FBTyxZQUFZLENBQUMsSUFBSSxDQUFDLG1CQUFtQixPQUFPLDZCQUE2QixFQUFFLEVBQUUsS0FBSyxFQUFFLEtBQUssRUFBRSxDQUFDLENBQUM7SUFDdEgsQ0FBQztJQUNELElBQUksQ0FBQztRQUNILE9BQU8sT0FBTyxDQUFDLE1BQU0sQ0FBQyxPQUFPLElBQUksS0FBSyxRQUFRLENBQUMsQ0FBQyxDQUFDLFlBQVksQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDLENBQUMsSUFBSSxDQUFDLENBQUM7SUFDOUUsQ0FBQztJQUFDLE9BQU8sS0FBSyxFQUFFLENBQUM7UUFDZixNQUFNLElBQUksS0FBSyxDQUFDLE9BQU8sWUFBWSxDQUFDLElBQUksQ0FBQyxpQkFBaUIsT0FBTyxRQUFRLEVBQUUsRUFBRSxLQUFLLEVBQUUsS0FBSyxFQUFFLENBQUMsQ0FBQztJQUMvRixDQUFDO0FBQ0gsQ0FBQztBQUVELG1HQUFtRztBQUNuRyxNQUFNLFVBQVUsc0JBQXNCLENBQUMsSUFBNEQ7SUFDakcsTUFBTSxrQkFBa0IsR0FBRyxJQUFJLENBQUMsR0FBRyxDQUFDLENBQUMsRUFBRSxHQUFHLEtBQUssQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxRQUFRLENBQUMsTUFBTSxDQUFDLEVBQUUsQ0FBQyxLQUFLLEVBQUUsRUFBRSxDQUFDLEtBQUssQ0FBQyxDQUFDLENBQUMsQ0FBQyxNQUFNLENBQUMsQ0FBQyxDQUFDO0lBQzlHLE1BQU0sS0FBSyxHQUFHLEdBQUcsQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLEdBQUcsQ0FBQyxDQUFDLEVBQUUsa0JBQWtCLEdBQUcsQ0FBQyxDQUFDLENBQUMsQ0FBQztJQUM5RCxNQUFNLE9BQU8sR0FBRyxJQUFJLENBQUMsUUFBUSxLQUFLLFNBQVM7UUFDekMsQ0FBQyxDQUFDLGtCQUFrQixJQUFJLENBQUMsU0FBUyxJQUFJO1FBQ3RDLENBQUMsQ0FBQyxpQkFBaUIsSUFBSSxDQUFDLFFBQVEsS0FBSyxJQUFJLENBQUMsU0FBUyxJQUFJLENBQUM7SUFDMUQsTUFBTSxJQUFJLEdBQUcsSUFBSSxDQUFDLElBQUksQ0FBQyxRQUFRLENBQUMsSUFBSSxDQUFDLENBQUMsQ0FBQyxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDLEdBQUcsSUFBSSxDQUFDLElBQUksSUFBSSxDQUFDO0lBQ3JFLE9BQU8sR0FBRyxPQUFPLEtBQUssS0FBSyxLQUFLLElBQUksR0FBRyxLQUFLLEVBQUUsQ0FBQztBQUNqRCxDQUFDO0FBRUQsU0FBUyxjQUFjLENBQUMsSUFBc0I7SUFDNUMsSUFBSSxJQUFJLENBQUMsSUFBSSxLQUFLLE1BQU07UUFBRSxPQUFPLElBQUksQ0FBQztJQUN0QyxJQUFJLElBQVksQ0FBQztJQUNqQixJQUFJLElBQUksQ0FBQyxJQUFJLENBQUMsSUFBSSxLQUFLLE1BQU0sRUFBRSxDQUFDO1FBQzlCLElBQUksR0FBRyxJQUFJLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQztJQUN4QixDQUFDO1NBQU0sSUFBSSxJQUFJLENBQUMsSUFBSSxDQUFDLElBQUksS0FBSyxNQUFNLElBQUksZUFBZSxDQUFDLElBQUksQ0FBQyxTQUFTLENBQUMsRUFBRSxDQUFDO1FBQ3hFLElBQUksR0FBRyxVQUFVLENBQUMsSUFBSSxFQUFFLElBQUksQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLENBQUM7SUFDMUMsQ0FBQztTQUFNLENBQUM7UUFDTixPQUFPLElBQUksQ0FBQztJQUNkLENBQUM7SUFDRCxPQUFPO1FBQ0wsSUFBSSxFQUFFLE1BQU07UUFDWixJQUFJLEVBQUUsc0JBQXNCLENBQUM7WUFDM0IsR0FBRyxDQUFDLElBQUksQ0FBQyxRQUFRLEtBQUssU0FBUyxDQUFDLENBQUMsQ0FBQyxFQUFFLENBQUMsQ0FBQyxDQUFDLEVBQUUsUUFBUSxFQUFFLElBQUksQ0FBQyxRQUFRLEVBQUUsQ0FBQztZQUNuRSxTQUFTLEVBQUUsSUFBSSxDQUFDLFNBQVM7WUFDekIsSUFBSTtTQUNMLENBQUM7UUFDRixHQUFHLENBQUMsSUFBSSxDQUFDLGVBQWUsS0FBSyxTQUFTLENBQUMsQ0FBQyxDQUFDLEVBQUUsQ0FBQyxDQUFDLENBQUMsRUFBRSxlQUFlLEVBQUUsSUFBSSxDQUFDLGVBQWUsRUFBRSxDQUFDO0tBQ3pGLENBQUM7QUFDSixDQUFDO0FBRUQ7Ozs7Ozs7R0FPRztBQUNILE1BQU0sVUFBVSw0QkFBNEI7SUFDMUMsT0FBTztRQUNMLG9CQUFvQixFQUFFLElBQUk7UUFDMUIsZUFBZSxFQUFFLEtBQUssRUFBRSxFQUFFLE1BQU0sRUFBRSxFQUFFLEVBQUU7WUFDcEMsTUFBTSxXQUFXLEdBQUcsTUFBTSxDQUFDLE1BQU0sQ0FBQyxJQUFJLENBQUMsQ0FBQyxPQUFPLEVBQUUsRUFBRSxDQUFDLE9BQU8sQ0FBQyxJQUFJLEtBQUssTUFBTSxJQUFJLE9BQU8sQ0FBQyxPQUFPLENBQUMsSUFBSSxDQUFDLENBQUMsSUFBSSxFQUFFLEVBQUUsQ0FDM0csSUFBSSxDQUFDLElBQUksS0FBSyxNQUFNLElBQUksQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLElBQUksS0FBSyxNQUFNLElBQUksQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLElBQUksS0FBSyxNQUFNLElBQUksZUFBZSxDQUFDLElBQUksQ0FBQyxTQUFTLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFDO1lBQzFILElBQUksQ0FBQyxXQUFXO2dCQUFFLE9BQU8sTUFBTSxDQUFDO1lBQ2hDLE9BQU87Z0JBQ0wsR0FBRyxNQUFNO2dCQUNULE1BQU0sRUFBRSxNQUFNLENBQUMsTUFBTSxDQUFDLEdBQUcsQ0FBQyxDQUFDLE9BQU8sRUFBRSxFQUFFLENBQUMsT0FBTyxDQUFDLElBQUksS0FBSyxNQUFNO29CQUM1RCxDQUFDLENBQUMsRUFBRSxHQUFHLE9BQU8sRUFBRSxPQUFPLEVBQUUsT0FBTyxDQUFDLE9BQU8sQ0FBQyxHQUFHLENBQUMsY0FBYyxDQUFDLEVBQUU7b0JBQzlELENBQUMsQ0FBQyxPQUFPLENBQUM7YUFDYixDQUFDO1FBQ0osQ0FBQztLQUNGLENBQUM7QUFDSixDQUFDIn0=

package/package.json

@@ -2,7 +2,7 @@
   "author": "Task Venture Capital GmbH",
   "license": "MIT",
   "name": "@modelprofile.com/flexharness-models",
-  "version": "10.5.0",
+  "version": "10.7.0",
   "type": "module",
   "description": "Provider-independent model contracts, prompt caching and explicit model registration.",
   "main": "./dist_ts_models/index.js",

package/readme.md

@@ -40,6 +40,14 @@ most 64 characters, and never when it looks like a credential: a key or token pr
 `sk_`, `pk_`, `ghp_` or `eyJ`, or a long run of letters and digits. The description follows `cause`
 and the last attempt of a retried call.
 
+`createTextFilePartMiddleware()` sends the text files of user messages as text parts, for
+providers that read only images and PDFs as files. A file part with inline text, or whose media
+type names text (`isTextMediaType`: `text/*`, JSON, XML, YAML, CSV, script types, `+json`, `+xml`
+and `+yaml` types, and any type with a `charset`), becomes one text part with the file name, the
+media type and the content decoded in the stated charset (UTF-8 by default), in a fence longer
+than any backtick run in the content. Images, PDFs, other binary files, URLs and file references
+pass unchanged; a file that names text but does not decode fails before the request is sent.
+
 ## License and Legal Information
 
 This repository contains open-source code licensed under the MIT License. A copy of the license can be found in the repository license file.

package/ts_models/index.ts

@@ -3,5 +3,6 @@ export * from './classes.modelregistry.js';
 export * from './smartai.cache.js';
 export * from './smartai.limits.js';
 export * from './smartai.failures.js';
+export * from './smartai.textfiles.js';
 export { generateText, streamText, tool, jsonSchema } from './plugins.js';
 export type { ModelMessage, ToolSet, StreamTextResult } from './plugins.js';

package/ts_models/readme.md

@@ -40,6 +40,14 @@ most 64 characters, and never when it looks like a credential: a key or token pr
 `sk_`, `pk_`, `ghp_` or `eyJ`, or a long run of letters and digits. The description follows `cause`
 and the last attempt of a retried call.
 
+`createTextFilePartMiddleware()` sends the text files of user messages as text parts, for
+providers that read only images and PDFs as files. A file part with inline text, or whose media
+type names text (`isTextMediaType`: `text/*`, JSON, XML, YAML, CSV, script types, `+json`, `+xml`
+and `+yaml` types, and any type with a `charset`), becomes one text part with the file name, the
+media type and the content decoded in the stated charset (UTF-8 by default), in a fence longer
+than any backtick run in the content. Images, PDFs, other binary files, URLs and file references
+pass unchanged; a file that names text but does not decode fails before the request is sent.
+
 ## License and Legal Information
 
 This repository contains open-source code licensed under the MIT License. A copy of the license can be found in the repository license file.

package/ts_models/smartai.textfiles.ts

@@ -0,0 +1,141 @@
+import type * as plugins from './plugins.js';
+
+type TPromptMessage = plugins.LanguageModelV4Prompt[number];
+type TUserContentPart = Extract<TPromptMessage, { role: 'user' }>['content'][number];
+type TFilePart = Extract<TUserContentPart, { type: 'file' }>;
+
+interface IParsedMediaType {
+  essence: string;
+  charset?: string;
+}
+
+const textualApplicationTypes = new Set([
+  'application/csv',
+  'application/ecmascript',
+  'application/graphql',
+  'application/javascript',
+  'application/json',
+  'application/jsonl',
+  'application/ndjson',
+  'application/sql',
+  'application/toml',
+  'application/typescript',
+  'application/x-javascript',
+  'application/x-ndjson',
+  'application/x-sh',
+  'application/x-toml',
+  'application/x-typescript',
+  'application/x-yaml',
+  'application/xml',
+  'application/yaml',
+]);
+
+const textualSuffixes = ['+json', '+xml', '+yaml'];
+
+function parseMediaType(mediaType: string): IParsedMediaType {
+  const [essence = '', ...parameters] = mediaType.split(';');
+  let charset: string | undefined;
+  for (const parameter of parameters) {
+    const separator = parameter.indexOf('=');
+    if (separator === -1) continue;
+    if (parameter.slice(0, separator).trim().toLowerCase() !== 'charset') continue;
+    charset = parameter.slice(separator + 1).trim().replace(/^"(.*)"$/u, '$1');
+  }
+  return { essence: essence.trim().toLowerCase(), ...(charset ? { charset } : {}) };
+}
+
+/**
+ * Whether a media type names text: any `text/*` type, a JSON, XML, YAML, CSV, script or similar
+ * `application/*` type, a structured-syntax `+json`, `+xml` or `+yaml` type, or any type that
+ * states its `charset`. `text` alone, the top-level form a prompt may use, counts as text.
+ */
+export function isTextMediaType(mediaType: string): boolean {
+  const { essence, charset } = parseMediaType(mediaType);
+  if (charset !== undefined) return true;
+  if (essence === 'text' || essence.startsWith('text/')) return true;
+  if (textualApplicationTypes.has(essence)) return true;
+  return essence.startsWith('application/') && textualSuffixes.some((suffix) => essence.endsWith(suffix));
+}
+
+function decodeBase64(data: string): Uint8Array {
+  const binary = atob(data);
+  const bytes = new Uint8Array(binary.length);
+  for (let index = 0; index < binary.length; index++) bytes[index] = binary.charCodeAt(index);
+  return bytes;
+}
+
+function describeFile(part: TFilePart): string {
+  return part.filename === undefined ? `file part of media type ${part.mediaType}` : `file "${part.filename}"`;
+}
+
+function decodeText(part: TFilePart, data: Uint8Array | string): string {
+  const charset = parseMediaType(part.mediaType).charset ?? 'utf-8';
+  let decoder: TextDecoder;
+  try {
+    decoder = new TextDecoder(charset, { fatal: true });
+  } catch (error) {
+    throw new Error(`The ${describeFile(part)} names charset "${charset}", which cannot be decoded.`, { cause: error });
+  }
+  try {
+    return decoder.decode(typeof data === 'string' ? decodeBase64(data) : data);
+  } catch (error) {
+    throw new Error(`The ${describeFile(part)} is not valid ${charset} text.`, { cause: error });
+  }
+}
+
+/** The text a model reads for one text file: its name and media type, then its content, fenced. */
+export function formatTextFileForModel(file: { filename?: string; mediaType: string; text: string }): string {
+  const longestBacktickRun = Math.max(0, ...Array.from(file.text.matchAll(/`+/gu), (match) => match[0].length));
+  const fence = '`'.repeat(Math.max(3, longestBacktickRun + 1));
+  const heading = file.filename === undefined
+    ? `Attached file (${file.mediaType}):`
+    : `Attached file ${file.filename} (${file.mediaType}):`;
+  const body = file.text.endsWith('\n') ? file.text : `${file.text}\n`;
+  return `${heading}\n${fence}\n${body}${fence}`;
+}
+
+function inlineTextFile(part: TUserContentPart): TUserContentPart {
+  if (part.type !== 'file') return part;
+  let text: string;
+  if (part.data.type === 'text') {
+    text = part.data.text;
+  } else if (part.data.type === 'data' && isTextMediaType(part.mediaType)) {
+    text = decodeText(part, part.data.data);
+  } else {
+    return part;
+  }
+  return {
+    type: 'text',
+    text: formatTextFileForModel({
+      ...(part.filename === undefined ? {} : { filename: part.filename }),
+      mediaType: part.mediaType,
+      text,
+    }),
+    ...(part.providerOptions === undefined ? {} : { providerOptions: part.providerOptions }),
+  };
+}
+
+/**
+ * Sends the text files of user messages as text parts, for providers that read only images and
+ * PDFs as files. A file part whose data is inline text, or whose media type names text (see
+ * `isTextMediaType`), becomes one text part that carries the file name, the media type and the
+ * content decoded in the stated charset, UTF-8 by default. Images, PDFs, other binary files, URLs
+ * and provider file references are passed on unchanged. A file that names text but does not decode
+ * fails the call before the request is sent.
+ */
+export function createTextFilePartMiddleware(): plugins.LanguageModelV4Middleware {
+  return {
+    specificationVersion: 'v4',
+    transformParams: async ({ params }) => {
+      const hasTextFile = params.prompt.some((message) => message.role === 'user' && message.content.some((part) =>
+        part.type === 'file' && (part.data.type === 'text' || (part.data.type === 'data' && isTextMediaType(part.mediaType)))));
+      if (!hasTextFile) return params;
+      return {
+        ...params,
+        prompt: params.prompt.map((message) => message.role === 'user'
+          ? { ...message, content: message.content.map(inlineTextFile) }
+          : message),
+      };
+    },
+  };
+}