@manta-eu/upload 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Manta AI AS
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,72 @@
+# @manta-eu/upload
+
+A CLI that uploads files to a Manta datastore from outside the app: a script, a scheduled job, or
+another system's export. It uses the upload flow the chat uses, so the file lands in the datastore
+exactly as a chat upload would.
+
+CSV, Parquet, Turtle and N-Triples files are then ingested into the datastore's knowledge graph.
+Any other type is stored but not ingested.
+
+## Use
+
+```shell
+pnpm manta-upload people.csv orders.parquet \
+  --api-url https://app.example \
+  --datastore salthouse \
+  --keycloak-url https://keycloak.example \
+  --keycloak-realm your-realm \
+  --client-id your-realm-app
+```
+
+For each file the CLI prints the file's IRI, its media type and its size.
+
+## Log in
+
+Login works as it does for `manta-codegen`, whose token code this package reuses.
+`MANTA_CLIENT_SECRET` decides the path.
+
+**No secret: a person.** The CLI starts the OAuth device flow with PKCE, prints a link and a code,
+then waits for the browser. The upload is attributed to that person. The CLI keeps the refresh token,
+so a long batch renews its access token silently; it asks for a new browser login only if Keycloak
+refuses the refresh token, for example once the session's own lifetime ends.
+
+**A secret: a machine.** The CLI asks for a service account token. Add `--scope` for a narrower
+token.
+
+```shell
+export MANTA_CLIENT_SECRET=...   # never a flag: an argument list is public on the machine
+```
+
+A service account only gets a datastore today if it holds the `operator` realm role. See the
+`manta-codegen` README for that trap and for the `azp` one.
+
+## What the CLI does
+
+- **Integrity.** The CLI hashes the bytes it uploads and aborts if the file changes while it's being
+  read. A part that storage answers with a 5xx, a 429 or a 408, or whose connection drops, is retried
+  for up to three attempts with backoff; a refusal such as a 403 is not.
+
+## What the API checks
+
+- **Access.** The API refuses the upload if the token may not use `--datastore`. It never falls back
+  to another datastore.
+- **Content.** For the four ingested types, the API reads the start of the file, and for Parquet the
+  end too. It refuses and deletes a file whose bytes contradict its extension: a `.parquet` without
+  Parquet's magic bytes, or a `.csv`, `.ttl` or `.nt` that isn't UTF-8 text. Re-save a CSV exported as
+  Windows-1252 or ISO-8859-1 as UTF-8 first.
+- **A check that can't run.** If the API cannot read the file back to check it (a storage error, say),
+  it records the file anyway and leaves the verdict to the ingest.
+- **Size.** The API accepts files up to 5 GiB. The ingest skips a file over its own limit, 512 MiB by
+  default.
+
+## How ingesting went
+
+The ingest runs in batches, a few minutes after upload. Each version of a file it handles gets a
+**File Ingest** (`ingest:IngestRun`) node in the datastore:
+
+- `prov:used` points to the file;
+- `ingest:status` is `ingested`, `unparsable`, `gone` or `too_large`, with `prov:endedAtTime`;
+- on success, `ingest:recordsRead`, `ingest:recordsDropped` and `prov:generated`, the dataset the
+  rows belong to.
+
+A file whose key can't be written as an IRI gets no node. Agents never mints such a key.

package/dist/cli.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+export {};
+//# sourceMappingURL=cli.d.ts.map

package/dist/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":""}

package/dist/cli.js

@@ -0,0 +1,58 @@
+#!/usr/bin/env node
+import { cachedToken, describeError, deviceLoginTokenSource, mintWithClientCredentials, } from "@manta-eu/codegen";
+import { Command } from "commander";
+import { graphqlClient, graphqlEndpoint } from "./graphql.js";
+import { uploadFile } from "./upload.js";
+// The secret stays out of the flags: an argument list is readable to every process on the machine.
+function tokenSourceFor(options) {
+    const client = { baseUrl: options.keycloakUrl, realm: options.keycloakRealm, clientId: options.clientId };
+    const secret = process.env.MANTA_CLIENT_SECRET;
+    if (secret) {
+        return cachedToken(() => mintWithClientCredentials({ ...client, clientSecret: secret, scope: options.scope }));
+    }
+    // Refreshes silently, so a long batch does not stop for a second browser login.
+    return cachedToken(deviceLoginTokenSource(client, (login) => console.error(`manta-upload: open ${login.verificationUri} and enter the code ${login.userCode}`)));
+}
+async function upload(paths, options) {
+    const graphql = graphqlClient({
+        endpoint: graphqlEndpoint(options.apiUrl),
+        datastore: options.datastore,
+        token: tokenSourceFor(options),
+    });
+    let failed = 0;
+    // A failed file does not stop the rest of the batch; the exit code reports it at the end.
+    for (const path of paths) {
+        try {
+            const uploaded = await uploadFile({ path, graphql });
+            console.error(`manta-upload: ${path} -> ${uploaded.fileIri} (${uploaded.mediaType}, ${uploaded.byteSize} bytes)`);
+        }
+        catch (error) {
+            failed += 1;
+            console.error(`manta-upload: ${path} failed: ${describeError(error)}`);
+        }
+    }
+    if (failed > 0) {
+        console.error(`manta-upload: ${failed} of ${paths.length} file(s) failed`);
+        process.exitCode = 1;
+    }
+}
+const program = new Command()
+    .name("manta-upload")
+    .description("Upload files to a Manta datastore. CSV, Parquet, Turtle and N-Triples files are ingested into its knowledge graph.")
+    .argument("<files...>", "files to upload")
+    .requiredOption("--api-url <url>", "Manta API base URL")
+    .requiredOption("--datastore <id>", "datastore to upload into")
+    .requiredOption("--keycloak-url <url>", "Keycloak base URL")
+    .requiredOption("--keycloak-realm <realm>", "Keycloak realm")
+    .requiredOption("--client-id <id>", "Keycloak client")
+    .option("--scope <scope>", "scope to ask a service account token for")
+    .action(async (paths, options) => {
+    try {
+        await upload(paths, options);
+    }
+    catch (error) {
+        console.error(describeError(error));
+        process.exitCode = 1;
+    }
+});
+await program.parseAsync();

package/dist/graphql.d.ts

@@ -0,0 +1,10 @@
+import { type FetchLike } from "@manta-eu/codegen";
+export declare function graphqlEndpoint(apiUrl: string): string;
+export type GraphqlClient = <T>(query: string, variables: Record<string, unknown>) => Promise<T>;
+export declare function graphqlClient(options: {
+    endpoint: string;
+    datastore: string;
+    token: () => Promise<string>;
+    fetchImpl?: FetchLike;
+}): GraphqlClient;
+//# sourceMappingURL=graphql.d.ts.map

package/dist/graphql.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"graphql.d.ts","sourceRoot":"","sources":["../src/graphql.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiB,KAAK,SAAS,EAAY,MAAM,mBAAmB,CAAC;AAE5E,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAEtD;AAED,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC;AAGjG,wBAAgB,aAAa,CAAC,OAAO,EAAE;IACrC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IAC7B,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB,GAAG,aAAa,CAqChB"}

package/dist/graphql.js

@@ -0,0 +1,33 @@
+import { describeError, readJson } from "@manta-eu/codegen";
+export function graphqlEndpoint(apiUrl) {
+    return `${apiUrl.replace(/\/$/, "")}/graphql`;
+}
+// The datastore travels as a header: the API refuses the call rather than fall back to another one.
+export function graphqlClient(options) {
+    const { endpoint, datastore, token, fetchImpl = fetch } = options;
+    return async (query, variables) => {
+        // Minted outside the try: a login that fails is not the API being unreachable.
+        const authorization = `Bearer ${await token()}`;
+        let response;
+        try {
+            response = await fetchImpl(endpoint, {
+                method: "POST",
+                headers: { "content-type": "application/json", authorization, "x-manta-datastore": datastore },
+                body: JSON.stringify({ query, variables }),
+            });
+        }
+        catch (cause) {
+            throw new Error(`manta-upload: cannot reach ${endpoint}. ${describeError(cause)}`, { cause });
+        }
+        const body = await readJson(response, endpoint, "manta-upload");
+        if (body.errors?.length) {
+            throw new Error(`manta-upload: ${body.errors.map((error) => error.message).join("; ")}`);
+        }
+        if (!response.ok || !body.data) {
+            // The Worker's auth middleware answers before Yoga, so its reason is `error`, not `errors`.
+            const reason = typeof body.error === "string" ? `: ${body.error}` : " without data";
+            throw new Error(`manta-upload: ${endpoint} answered status ${response.status}${reason}`);
+        }
+        return body.data;
+    };
+}

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+export { graphqlClient, graphqlEndpoint, type GraphqlClient } from "./graphql.js";
+export { readFully, uploadFile, uploadPartSize, type PartReader, type Sleep, type UploadedFile } from "./upload.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,KAAK,aAAa,EAAE,MAAM,cAAc,CAAC;AAClF,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,cAAc,EAAE,KAAK,UAAU,EAAE,KAAK,KAAK,EAAE,KAAK,YAAY,EAAE,MAAM,aAAa,CAAC"}

package/dist/index.js

@@ -0,0 +1,2 @@
+export { graphqlClient, graphqlEndpoint } from "./graphql.js";
+export { readFully, uploadFile, uploadPartSize } from "./upload.js";

package/dist/upload.d.ts

@@ -0,0 +1,23 @@
+import { type FetchLike } from "@manta-eu/codegen";
+import type { GraphqlClient } from "./graphql.js";
+export declare const uploadPartSize: number;
+export type UploadedFile = {
+    fileIri: string;
+    mediaType: string;
+    storageKey: string;
+    byteSize: number;
+};
+export type PartReader = {
+    read(buffer: Uint8Array, offset: number, length: number, position: number): Promise<{
+        bytesRead: number;
+    }>;
+};
+export declare function readFully(file: PartReader, start: number, length: number): Promise<Uint8Array>;
+export type Sleep = (ms: number) => Promise<void>;
+export declare function uploadFile(options: {
+    path: string;
+    graphql: GraphqlClient;
+    fetchImpl?: FetchLike;
+    sleep?: Sleep;
+}): Promise<UploadedFile>;
+//# sourceMappingURL=upload.d.ts.map

package/dist/upload.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"upload.d.ts","sourceRoot":"","sources":["../src/upload.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,KAAK,SAAS,EAAqB,MAAM,mBAAmB,CAAC;AACtE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAGlD,eAAO,MAAM,cAAc,QAAkB,CAAC;AAoB9C,MAAM,MAAM,YAAY,GAAG;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC;AAExG,MAAM,MAAM,UAAU,GAAG;IACvB,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC5G,CAAC;AAGF,wBAAsB,SAAS,CAAC,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAapG;AAED,MAAM,MAAM,KAAK,GAAG,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AAiDlD,wBAAsB,UAAU,CAAC,OAAO,EAAE;IACxC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,aAAa,CAAC;IACvB,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,KAAK,CAAC,EAAE,KAAK,CAAC;CACf,GAAG,OAAO,CAAC,YAAY,CAAC,CASxB"}

package/dist/upload.js

@@ -0,0 +1,109 @@
+import { createHash } from "node:crypto";
+import { open } from "node:fs/promises";
+import { basename } from "node:path";
+import { isTransientStatus } from "@manta-eu/codegen";
+// Must equal agents' uploadPartSize: the API presigns one part per this many bytes.
+export const uploadPartSize = 8 * 1024 * 1024;
+const initiateMutation = `mutation Initiate($input: InitiateUploadInput!) {
+  initiateUpload(input: $input) { uploadId storageKey partUrls { partNumber url } }
+}`;
+const completeMutation = `mutation Complete($input: CompleteUploadInput!) {
+  completeUpload(input: $input) { fileIri mediaType }
+}`;
+const abortMutation = `mutation Abort($input: AbortUploadInput!) {
+  abortUpload(input: $input) { ok }
+}`;
+// One read may return fewer bytes than asked for, so read until the part is full or the file ends.
+export async function readFully(file, start, length) {
+    const buffer = new Uint8Array(length);
+    let filled = 0;
+    while (filled < length) {
+        const { bytesRead } = await file.read(buffer, filled, length - filled, start + filled);
+        if (bytesRead === 0)
+            break;
+        filled += bytesRead;
+    }
+    return buffer.subarray(0, filled);
+}
+const partAttempts = 3;
+const partBackoffMs = 1_000;
+const sleepFor = (ms) => new Promise((done) => setTimeout(done, ms));
+// Only what storage has finally refused, so an unrecognized failure stays retryable.
+class NonRetryablePartError extends Error {
+}
+// Overload, a timeout or a dropped connection is transient for S3-compatible stores: the part is retried, not the file.
+async function putPart(url, body, fetchImpl, sleep) {
+    for (let attempt = 1;; attempt++) {
+        try {
+            return await putPartOnce(url, body, fetchImpl);
+        }
+        catch (error) {
+            const retryable = !(error instanceof NonRetryablePartError);
+            if (!retryable || attempt >= partAttempts)
+                throw error;
+            await sleep(partBackoffMs * 2 ** (attempt - 1));
+        }
+    }
+}
+async function putPartOnce(url, body, fetchImpl) {
+    const response = await fetchImpl(url, { method: "PUT", body });
+    const etag = response.headers.get("etag");
+    if (!response.ok) {
+        // The storage error document is the only thing that says why a part was refused.
+        const detail = (await response.text().catch(() => "")).slice(0, 500);
+        const message = `manta-upload: storage refused a part with status ${response.status}: ${detail}`;
+        throw isTransientStatus(response.status) ? new Error(message) : new NonRetryablePartError(message);
+    }
+    // The part is already stored, so a body that cannot be discarded must not cost a re-PUT.
+    await response.body?.cancel().catch(() => undefined);
+    // The part is stored, so repeating the PUT cannot conjure the header completeUpload needs.
+    if (!etag) {
+        throw new NonRetryablePartError("manta-upload: storage stored a part but answered without an ETag");
+    }
+    return etag;
+}
+// Parts go up one at a time, so an upload holds at most one part in memory.
+export async function uploadFile(options) {
+    // One handle for the whole upload: an export that lands by rename cannot swap the file under the loop.
+    const file = await open(options.path, "r");
+    try {
+        return await uploadHandle(file, { ...options, fileName: basename(options.path) });
+    }
+    finally {
+        await file.close();
+    }
+}
+async function uploadHandle(file, options) {
+    const { path, fileName, graphql, fetchImpl = fetch, sleep = sleepFor } = options;
+    const byteSize = (await file.stat()).size;
+    if (byteSize === 0)
+        throw new Error(`manta-upload: ${path} is empty`);
+    const { initiateUpload } = await graphql(initiateMutation, { input: { fileName, byteSize } });
+    const { uploadId, storageKey, partUrls } = initiateUpload;
+    try {
+        if (partUrls.length !== Math.ceil(byteSize / uploadPartSize)) {
+            throw new Error(`manta-upload: the API split ${fileName} into ${partUrls.length} parts, which this client cannot fill`);
+        }
+        const parts = [];
+        // Hashed from the bytes that went up, so the hash describes the stored object.
+        const hash = createHash("sha256");
+        for (const { partNumber, url } of partUrls) {
+            const start = (partNumber - 1) * uploadPartSize;
+            const body = await readFully(file, start, uploadPartSize);
+            if (body.length !== Math.min(uploadPartSize, byteSize - start)) {
+                throw new Error(`manta-upload: ${fileName} changed while it was being read`);
+            }
+            hash.update(body);
+            parts.push({ partNumber, etag: await putPart(url, body, fetchImpl, sleep) });
+        }
+        const facts = { byteSize, sha256: hash.digest("hex") };
+        const { completeUpload } = await graphql(completeMutation, {
+            input: { uploadId, storageKey, fileName, parts, facts },
+        });
+        return { ...completeUpload, storageKey, byteSize };
+    }
+    catch (error) {
+        await graphql(abortMutation, { input: { uploadId, storageKey } }).catch(() => undefined);
+        throw error;
+    }
+}

package/package.json

@@ -0,0 +1,43 @@
+{
+  "name": "@manta-eu/upload",
+  "version": "0.1.0",
+  "description": "CLI that uploads files to a Manta datastore, where supported types are ingested into its knowledge graph",
+  "type": "module",
+  "bin": {
+    "manta-upload": "./dist/cli.js"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "author": "",
+  "license": "MIT",
+  "dependencies": {
+    "commander": "^15.0.0",
+    "@manta-eu/codegen": "0.1.0"
+  },
+  "devDependencies": {
+    "@types/node": "^24.13.3",
+    "@typescript-eslint/eslint-plugin": "^8.56.1",
+    "@typescript-eslint/parser": "^8.56.1",
+    "eslint": "^9.39.2",
+    "tsx": "^4.20.6",
+    "typescript": "^5.0.0",
+    "vitest": "^3.0.0"
+  },
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "files": [
+    "dist"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json",
+    "test": "vitest --run",
+    "types": "tsc --noEmit"
+  }
+}