@datalayer/core 1.1.60 → 1.1.66

package/lib/api/contents/dataservers.js

@@ -0,0 +1,121 @@
+/*
+ * Copyright (c) 2023-2025 Datalayer, Inc.
+ * Distributed under the terms of the Modified BSD License.
+ */
+/*
+ * Copyright (c) 2023-2026 Datalayer, Inc.
+ * Distributed under the terms of the Modified BSD License.
+ */
+/**
+ * A Dataserver registration through the service: read, moved, re-identified.
+ *
+ * The gateway itself runs in a customer's network. What is here is what it
+ * last said of itself — a heartbeat lease, the connectors it advertises,
+ * its queue — and the levers the catalog has on it: drain, resume, revoke,
+ * and a certificate issued or rotated from a CSR the gateway made, so the
+ * private key never travels.
+ *
+ * The wire types are the generated ones; what is declared here is what the
+ * contract does not name — the state union as a standalone type, its labels,
+ * and which action a reader may take on a gateway in a given state.
+ */
+import { requestDatalayerAPI } from '../DatalayerApi.js';
+import { API_BASE_PATHS, DEFAULT_SERVICE_URLS } from '../constants.js';
+import { contentsToCamelCase } from '../../models/contents/index.js';
+export const DATASERVER_STATES = [
+    'registering',
+    'ready',
+    'degraded',
+    'unavailable',
+    'draining',
+    'revoked',
+];
+/** The state as a person reads it. */
+export const DATASERVER_STATE_LABELS = {
+    registering: 'Registering',
+    ready: 'Online',
+    degraded: 'Degraded',
+    unavailable: 'Offline',
+    draining: 'Draining',
+    revoked: 'Revoked',
+};
+/**
+ * Which actions a reader may take on a gateway in this state, and why not.
+ *
+ * `view` inspects; `execute` tests; owner or update-holder registers,
+ * rotates, drains, resumes, revokes, archives and deletes. A gateway that is
+ * revoked, or one that has gone quiet, keeps its record — the disabled
+ * actions name the state as the reason rather than disappearing.
+ */
+export const dataserverActionAvailability = (state, permissions, options = {}) => {
+    const manages = permissions.isOwner || permissions.update;
+    const needsManage = 'Only the owner or an update-holder may do this.';
+    const revoked = state === 'revoked';
+    const decide = (allowed, permissionReason, stateReason) => {
+        if (!allowed) {
+            return { enabled: false, reason: permissionReason };
+        }
+        if (stateReason) {
+            return { enabled: false, reason: stateReason };
+        }
+        return { enabled: true };
+    };
+    return {
+        test: decide(permissions.execute || manages, 'Only someone who may execute through this gateway can test it.', revoked ? 'The gateway is revoked; its identity is no longer accepted.' : undefined),
+        rotateIdentity: decide(manages, needsManage, revoked ? 'A revoked gateway is re-admitted with a new identity, not a rotation.' : undefined),
+        drain: decide(manages, needsManage, revoked
+            ? 'The gateway is revoked.'
+            : state === 'draining'
+                ? 'The gateway is already draining.'
+                : state === 'registering' || state === 'unavailable' || !state
+                    ? 'Nothing routes to a gateway that is not online; there is nothing to drain.'
+                    : undefined),
+        resume: decide(manages, needsManage, revoked
+            ? 'The gateway is revoked.'
+            : state !== 'draining'
+                ? 'Only a draining gateway is resumed.'
+                : undefined),
+        revoke: decide(manages, needsManage, revoked ? 'The gateway is already revoked.' : undefined),
+        archive: decide(manages, needsManage, options.archived ? 'The registration is already archived.' : undefined),
+        delete: decide(manages, needsManage),
+    };
+};
+const convertResponse = (value) => contentsToCamelCase(value);
+const dataserverUrl = (baseUrl, sourceUid, suffix = '') => `${baseUrl}${API_BASE_PATHS.CONTENTS}/dataservers/${encodeURIComponent(sourceUid)}${suffix}`;
+/** The gateway as last heard: its lease, its connectors, its queue. */
+export const getDataserverStatus = async (token, sourceUid, baseUrl = DEFAULT_SERVICE_URLS.CONTENTS) => convertResponse(await requestDatalayerAPI({
+    url: dataserverUrl(baseUrl, sourceUid, '/status'),
+    method: 'GET',
+    token,
+}));
+/** Try the gateway on Flight and on the HTTPS fallback; the answer is a verdict per path. */
+export const testDataserver = async (token, sourceUid, baseUrl = DEFAULT_SERVICE_URLS.CONTENTS) => convertResponse(await requestDatalayerAPI({
+    url: dataserverUrl(baseUrl, sourceUid, '/test'),
+    method: 'POST',
+    token,
+}));
+const transition = async (token, sourceUid, action, baseUrl) => convertResponse(await requestDatalayerAPI({
+    url: dataserverUrl(baseUrl, sourceUid, `/${action}`),
+    method: 'POST',
+    token,
+}));
+/** Stop routing new queries; the ones running finish. */
+export const drainDataserver = (token, sourceUid, baseUrl = DEFAULT_SERVICE_URLS.CONTENTS) => transition(token, sourceUid, 'drain', baseUrl);
+/** Route to the gateway again. */
+export const resumeDataserver = (token, sourceUid, baseUrl = DEFAULT_SERVICE_URLS.CONTENTS) => transition(token, sourceUid, 'resume', baseUrl);
+/** Refuse the gateway's identity from now on. Nothing in its network changes. */
+export const revokeDataserver = (token, sourceUid, baseUrl = DEFAULT_SERVICE_URLS.CONTENTS) => transition(token, sourceUid, 'revoke', baseUrl);
+/** A first certificate for the identity the CSR names. */
+export const issueDataserverIdentity = async (token, sourceUid, request, baseUrl = DEFAULT_SERVICE_URLS.CONTENTS) => convertResponse(await requestDatalayerAPI({
+    url: dataserverUrl(baseUrl, sourceUid, '/identity'),
+    method: 'POST',
+    token,
+    body: { csr: request.csr },
+}));
+/** A new certificate that overlaps the current one, so nothing stops. */
+export const rotateDataserverIdentity = async (token, sourceUid, request, baseUrl = DEFAULT_SERVICE_URLS.CONTENTS) => convertResponse(await requestDatalayerAPI({
+    url: dataserverUrl(baseUrl, sourceUid, '/identity/rotate'),
+    method: 'POST',
+    token,
+    body: { csr: request.csr },
+}));

package/lib/api/contents/datasets.d.ts

@@ -0,0 +1,15 @@
+import type { DatasetPublication, DatasetPublicationCreate, DatasetPublicationList, DatasetRevision, DatasetRevisionCreate, DatasetRevisionList } from './generated';
+export declare const createDatasetRevision: (token: string, sourceUid: string, request: DatasetRevisionCreate, idempotencyKey: string, baseUrl?: string) => Promise<DatasetRevision>;
+export declare const listDatasetRevisions: (token: string, sourceUid: string, baseUrl?: string) => Promise<DatasetRevisionList>;
+export declare const getDatasetRevision: (token: string, sourceUid: string, revisionUid: string, baseUrl?: string) => Promise<DatasetRevision>;
+export declare const createDatasetPublication: (token: string, sourceUid: string, request: DatasetPublicationCreate, idempotencyKey: string, baseUrl?: string) => Promise<DatasetPublication>;
+export declare const listDatasetPublications: (token: string, sourceUid: string, baseUrl?: string) => Promise<DatasetPublicationList>;
+export declare const unpublishDataset: (token: string, sourceUid: string, publicationUid: string, baseUrl?: string) => Promise<DatasetPublication>;
+/**
+ * Every Dataset currently published, across sources.
+ *
+ * What the Library shows: a publication names one immutable revision of a
+ * Dataset, so this is the list of what has been made public, not of Datasets.
+ * Any signed-in caller may read it — published means public.
+ */
+export declare const listPublishedDatasets: (token: string, baseUrl?: string) => Promise<DatasetPublicationList>;

package/lib/api/contents/datasets.js

@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2023-2025 Datalayer, Inc.
+ * Distributed under the terms of the Modified BSD License.
+ */
+import { requestDatalayerAPI } from '../DatalayerApi.js';
+import { API_BASE_PATHS, DEFAULT_SERVICE_URLS } from '../constants.js';
+import { contentsToCamelCase, contentsToSnakeCase } from '../../models/contents/index.js';
+const convert = (value) => contentsToCamelCase(value);
+const resourceUrl = (baseUrl, sourceUid, resource, suffix = '') => `${baseUrl}${API_BASE_PATHS.CONTENTS}/sources/${encodeURIComponent(sourceUid)}/${resource}${suffix}`;
+export const createDatasetRevision = async (token, sourceUid, request, idempotencyKey, baseUrl = DEFAULT_SERVICE_URLS.CONTENTS) => convert(await requestDatalayerAPI({
+    url: resourceUrl(baseUrl, sourceUid, 'revisions'), method: 'POST', token,
+    headers: { 'Idempotency-Key': idempotencyKey },
+    body: contentsToSnakeCase(request),
+}));
+export const listDatasetRevisions = async (token, sourceUid, baseUrl = DEFAULT_SERVICE_URLS.CONTENTS) => convert(await requestDatalayerAPI({
+    url: resourceUrl(baseUrl, sourceUid, 'revisions'), method: 'GET', token,
+}));
+export const getDatasetRevision = async (token, sourceUid, revisionUid, baseUrl = DEFAULT_SERVICE_URLS.CONTENTS) => convert(await requestDatalayerAPI({
+    url: resourceUrl(baseUrl, sourceUid, 'revisions', `/${encodeURIComponent(revisionUid)}`),
+    method: 'GET', token,
+}));
+export const createDatasetPublication = async (token, sourceUid, request, idempotencyKey, baseUrl = DEFAULT_SERVICE_URLS.CONTENTS) => convert(await requestDatalayerAPI({
+    url: resourceUrl(baseUrl, sourceUid, 'publications'), method: 'POST', token,
+    headers: { 'Idempotency-Key': idempotencyKey },
+    body: contentsToSnakeCase(request),
+}));
+export const listDatasetPublications = async (token, sourceUid, baseUrl = DEFAULT_SERVICE_URLS.CONTENTS) => convert(await requestDatalayerAPI({
+    url: resourceUrl(baseUrl, sourceUid, 'publications'), method: 'GET', token,
+}));
+export const unpublishDataset = async (token, sourceUid, publicationUid, baseUrl = DEFAULT_SERVICE_URLS.CONTENTS) => convert(await requestDatalayerAPI({
+    url: resourceUrl(baseUrl, sourceUid, 'publications', `/${encodeURIComponent(publicationUid)}`),
+    method: 'DELETE', token,
+}));
+/**
+ * Every Dataset currently published, across sources.
+ *
+ * What the Library shows: a publication names one immutable revision of a
+ * Dataset, so this is the list of what has been made public, not of Datasets.
+ * Any signed-in caller may read it — published means public.
+ */
+export const listPublishedDatasets = async (token, baseUrl = DEFAULT_SERVICE_URLS.CONTENTS) => convert(await requestDatalayerAPI({
+    url: `${baseUrl}${API_BASE_PATHS.CONTENTS}/publications`,
+    method: 'GET',
+    token,
+}));

package/lib/api/contents/datasources.d.ts

@@ -0,0 +1,70 @@
+import type { CapabilityTicket, CapabilityTicketRequest, DatasetRevision, DatasourceCapabilities, DatasourceConfiguration, DatasourceQuery, DatasourceQueryCreate, DatasourceQueryList, DatasourceSchema, DatasourceTest, QuerySave } from './generated';
+export type DatasourceConnectorType = DatasourceConfiguration['connectorType'];
+export type DatasourceNetworkRoute = NonNullable<DatasourceConfiguration['networkRoute']>;
+export type DatasourceOperation = DatasourceCapabilities['operations'][number];
+export type DatasourceQueryStatus = DatasourceQuery['status'];
+/** The operations a Datasource may allow, in the order a form lists them. */
+export declare const DATASOURCE_OPERATIONS: ReadonlyArray<DatasourceOperation>;
+/**
+ * The connectors a person can pick in a form.
+ *
+ * `table` is not one of them: a published table is created by publishing it
+ * from a sandbox, and `POST /sources` refuses one made by hand
+ * (`PUBLISHED_TABLE_MANAGED`) because the record and its files have to come
+ * into being together. Saying that in the type keeps a creation form from
+ * having to invent labels and hints for something it cannot create.
+ */
+export type CreatableDatasourceConnectorType = Exclude<DatasourceConnectorType, 'table' | 'live'>;
+export declare const DATASOURCE_CONNECTOR_LABELS: Record<DatasourceConnectorType, string>;
+/** A query the service has finished with, one way or another. */
+export declare const DATASOURCE_QUERY_TERMINAL_STATUSES: ReadonlySet<DatasourceQueryStatus>;
+export declare const isDatasourceQueryTerminal: (status: DatasourceQueryStatus) => boolean;
+/** The bytes of a result, or a range of them, with what the headers said. */
+export interface DatasourceQueryResultBytes {
+    body: ArrayBuffer;
+    status: number;
+    headers: Record<string, string>;
+}
+/** Does the database answer through this source, right now? A verdict, not a stack trace. */
+export declare const testDatasource: (token: string, sourceUid: string, baseUrl?: string) => Promise<DatasourceTest>;
+/** The tables and columns the source exposes, as the service saw them. */
+export declare const discoverDatasourceSchema: (token: string, sourceUid: string, baseUrl?: string) => Promise<DatasourceSchema>;
+/** What the source may be asked, and how the answer can travel. */
+export declare const getDatasourceCapabilities: (token: string, sourceUid: string, baseUrl?: string) => Promise<DatasourceCapabilities>;
+/**
+ * Submit a statement; the answer is the job, not the rows.
+ *
+ * The service checks the statement against the source's operation allowlist
+ * and the limits against its policy before the job exists, so a refused
+ * query is refused here, with a reason, and not later on the stream.
+ */
+export declare const createDatasourceQuery: (token: string, sourceUid: string, request: DatasourceQueryCreate, idempotencyKey: string, baseUrl?: string) => Promise<DatasourceQuery>;
+/** The queries run against a source, newest first: its history. */
+export declare const listDatasourceQueries: (token: string, sourceUid: string, options?: {
+    cursor?: string;
+    limit?: number;
+}, baseUrl?: string) => Promise<DatasourceQueryList>;
+export declare const getDatasourceQuery: (token: string, queryUid: string, baseUrl?: string) => Promise<DatasourceQuery>;
+/** Ask for the query to stop; the cancellation reaches the connector. */
+export declare const cancelDatasourceQuery: (token: string, queryUid: string, baseUrl?: string) => Promise<DatasourceQuery>;
+/** Where a browser fetches a result from; a link, for a download. */
+export declare const datasourceQueryResultsUrl: (queryUid: string, baseUrl?: string) => string;
+/**
+ * The result of a finished query, as Arrow IPC bytes, or a `Range` of them.
+ *
+ * A range is how a stream that broke halfway is resumed rather than
+ * restarted — and how a preview reads the first bytes of a large result
+ * without the rest.
+ */
+export declare const downloadDatasourceQueryResults: (token: string, queryUid: string, options?: {
+    range?: string;
+}, baseUrl?: string) => Promise<DatasourceQueryResultBytes>;
+/**
+ * Keep a result: written into a Dataset as a verified revision.
+ *
+ * The bytes go from the service into the Dataset; nothing is downloaded to
+ * be uploaded again, and the answer is the revision that now holds them.
+ */
+export declare const saveDatasourceQueryAsDataset: (token: string, queryUid: string, request: QuerySave, baseUrl?: string) => Promise<DatasetRevision>;
+/** A Flight ticket for the result, for a client inside a sandbox. Never for the browser. */
+export declare const createDatasourceQueryTicket: (token: string, queryUid: string, request?: CapabilityTicketRequest, baseUrl?: string) => Promise<CapabilityTicket>;

package/lib/api/contents/datasources.js

@@ -0,0 +1,149 @@
+/*
+ * Copyright (c) 2023-2025 Datalayer, Inc.
+ * Distributed under the terms of the Modified BSD License.
+ */
+/*
+ * Copyright (c) 2023-2026 Datalayer, Inc.
+ * Distributed under the terms of the Modified BSD License.
+ */
+/**
+ * A Datasource through the service: tested, described, queried.
+ *
+ * Nothing here talks to the database. Contents does, with the credential it
+ * holds — directly, or through a Dataserver in the customer's network — and
+ * answers with a verdict, a schema, or a query job. A job is polled to its
+ * end; its result is Arrow IPC bytes read by range, and it can be kept as a
+ * Dataset revision rather than downloaded to be uploaded again.
+ *
+ * The wire types are the generated ones; what is declared here is what the
+ * contract does not name — the unions as standalone types, the labels a
+ * form shows, which statuses are terminal, and the bytes of a result.
+ */
+import { requestDatalayerAPI, requestDatalayerAPIWithResponse, } from '../DatalayerApi.js';
+import { API_BASE_PATHS, DEFAULT_SERVICE_URLS } from '../constants.js';
+import { contentsToCamelCase, contentsToSnakeCase, } from '../../models/contents/index.js';
+/** The operations a Datasource may allow, in the order a form lists them. */
+export const DATASOURCE_OPERATIONS = [
+    'select',
+    'describe',
+    'list',
+];
+export const DATASOURCE_CONNECTOR_LABELS = {
+    athena: 'Amazon Athena',
+    bigquery: 'Google BigQuery',
+    sql: 'SQL database',
+    // Listed, unlike above: a published table is a Datasource people see in
+    // the catalog and query like any other, even though no form makes one.
+    table: 'Published table',
+    // The same table while its kernel is up: answered from the live frame,
+    // and from the published snapshot once the kernel is gone.
+    live: 'Live table',
+};
+/** A query the service has finished with, one way or another. */
+export const DATASOURCE_QUERY_TERMINAL_STATUSES = new Set(['succeeded', 'failed', 'cancelled']);
+export const isDatasourceQueryTerminal = (status) => DATASOURCE_QUERY_TERMINAL_STATUSES.has(status);
+const convertResponse = (value) => contentsToCamelCase(value);
+const convertRequest = (value) => contentsToSnakeCase(value);
+const datasourceUrl = (baseUrl, sourceUid, suffix = '') => `${baseUrl}${API_BASE_PATHS.CONTENTS}/sources/${encodeURIComponent(sourceUid)}/datasource${suffix}`;
+const queryUrl = (baseUrl, queryUid, suffix = '') => `${baseUrl}${API_BASE_PATHS.CONTENTS}/queries/${encodeURIComponent(queryUid)}${suffix}`;
+/** Does the database answer through this source, right now? A verdict, not a stack trace. */
+export const testDatasource = async (token, sourceUid, baseUrl = DEFAULT_SERVICE_URLS.CONTENTS) => convertResponse(await requestDatalayerAPI({
+    url: datasourceUrl(baseUrl, sourceUid, '/test'),
+    method: 'POST',
+    token,
+}));
+/** The tables and columns the source exposes, as the service saw them. */
+export const discoverDatasourceSchema = async (token, sourceUid, baseUrl = DEFAULT_SERVICE_URLS.CONTENTS) => convertResponse(await requestDatalayerAPI({
+    url: datasourceUrl(baseUrl, sourceUid, '/schema'),
+    method: 'GET',
+    token,
+}));
+/** What the source may be asked, and how the answer can travel. */
+export const getDatasourceCapabilities = async (token, sourceUid, baseUrl = DEFAULT_SERVICE_URLS.CONTENTS) => convertResponse(await requestDatalayerAPI({
+    url: datasourceUrl(baseUrl, sourceUid, '/capabilities'),
+    method: 'GET',
+    token,
+}));
+/**
+ * Submit a statement; the answer is the job, not the rows.
+ *
+ * The service checks the statement against the source's operation allowlist
+ * and the limits against its policy before the job exists, so a refused
+ * query is refused here, with a reason, and not later on the stream.
+ */
+export const createDatasourceQuery = async (token, sourceUid, request, idempotencyKey, baseUrl = DEFAULT_SERVICE_URLS.CONTENTS) => convertResponse(await requestDatalayerAPI({
+    url: `${baseUrl}${API_BASE_PATHS.CONTENTS}/sources/${encodeURIComponent(sourceUid)}/queries`,
+    method: 'POST',
+    token,
+    headers: { 'Idempotency-Key': idempotencyKey },
+    body: convertRequest(request),
+}));
+/** The queries run against a source, newest first: its history. */
+export const listDatasourceQueries = async (token, sourceUid, options = {}, baseUrl = DEFAULT_SERVICE_URLS.CONTENTS) => {
+    const parameters = new URLSearchParams();
+    parameters.set('limit', String(options.limit ?? 50));
+    if (options.cursor) {
+        parameters.set('cursor', options.cursor);
+    }
+    return convertResponse(await requestDatalayerAPI({
+        url: `${baseUrl}${API_BASE_PATHS.CONTENTS}/sources/${encodeURIComponent(sourceUid)}/queries?${parameters.toString()}`,
+        method: 'GET',
+        token,
+    }));
+};
+export const getDatasourceQuery = async (token, queryUid, baseUrl = DEFAULT_SERVICE_URLS.CONTENTS) => convertResponse(await requestDatalayerAPI({
+    url: queryUrl(baseUrl, queryUid),
+    method: 'GET',
+    token,
+}));
+/** Ask for the query to stop; the cancellation reaches the connector. */
+export const cancelDatasourceQuery = async (token, queryUid, baseUrl = DEFAULT_SERVICE_URLS.CONTENTS) => convertResponse(await requestDatalayerAPI({
+    url: queryUrl(baseUrl, queryUid, '/cancel'),
+    method: 'POST',
+    token,
+}));
+/** Where a browser fetches a result from; a link, for a download. */
+export const datasourceQueryResultsUrl = (queryUid, baseUrl = DEFAULT_SERVICE_URLS.CONTENTS) => queryUrl(baseUrl, queryUid, '/results');
+/**
+ * The result of a finished query, as Arrow IPC bytes, or a `Range` of them.
+ *
+ * A range is how a stream that broke halfway is resumed rather than
+ * restarted — and how a preview reads the first bytes of a large result
+ * without the rest.
+ */
+export const downloadDatasourceQueryResults = async (token, queryUid, options = {}, baseUrl = DEFAULT_SERVICE_URLS.CONTENTS) => {
+    const response = await requestDatalayerAPIWithResponse({
+        url: datasourceQueryResultsUrl(queryUid, baseUrl),
+        method: 'GET',
+        token,
+        headers: options.range ? { Range: options.range } : undefined,
+        responseType: 'arraybuffer',
+    });
+    return {
+        body: response.data,
+        status: response.status,
+        headers: response.headers,
+    };
+};
+/**
+ * Keep a result: written into a Dataset as a verified revision.
+ *
+ * The bytes go from the service into the Dataset; nothing is downloaded to
+ * be uploaded again, and the answer is the revision that now holds them.
+ */
+export const saveDatasourceQueryAsDataset = async (token, queryUid, request, baseUrl = DEFAULT_SERVICE_URLS.CONTENTS) => convertResponse(await requestDatalayerAPI({
+    url: queryUrl(baseUrl, queryUid, '/save'),
+    method: 'POST',
+    token,
+    body: convertRequest({
+        ...request,
+        path: request.path.replace(/^\/+/, ''),
+    }),
+}));
+/** A Flight ticket for the result, for a client inside a sandbox. Never for the browser. */
+export const createDatasourceQueryTicket = async (token, queryUid, request = {}, baseUrl = DEFAULT_SERVICE_URLS.CONTENTS) => convertResponse(await requestDatalayerAPI({
+    url: queryUrl(baseUrl, queryUid, '/ticket'),
+    method: 'POST',
+    token,
+    body: convertRequest(request),
+}));