@unito/integration-sdk 1.0.4 → 1.0.6

package/.eslintrc.cjs

@@ -15,8 +15,7 @@ module.exports = {
       'rules': {
         'no-console': 'off',
         // Typescript already checks for this, with an easy exception on leading underscores names
-        "@typescript-eslint/no-unused-vars": 'off',
-        '@typescript-eslint/no-explicit-any': 'off'
+        '@typescript-eslint/no-unused-vars': 'off',
       }
     },
     {
@@ -24,6 +23,12 @@ module.exports = {
       'rules': {
         '@typescript-eslint/no-floating-promises': 'off'
       }
+    },
+    {
+      'files': ['**/*.ts'],
+      'rules': {
+        '@typescript-eslint/no-explicit-any': 'off',
+      }
     }
   ],
 };

package/dist/src/helpers.d.ts

@@ -0,0 +1,15 @@
+import { FieldSchema } from '@unito/integration-api';
+import { Filter } from './index.js';
+/**
+ * Use this helper function to retrieve the applicable filters from the context object. While using filters
+ * directly from context might work, it doesn't offer any guarantees about the shape of the filters nor the
+ * validity of the fields against which the filters are applied. On the other hand, this function ensures that
+ * all filters are valid and that the fields against which the filters are applied are present in the schema.
+ *
+ * @param context The object containing the raw filters
+ * @param fields The schema of the item against which the filters are applied
+ * @returns The validated filters
+ */
+export declare const getApplicableFilters: (context: {
+    filters: Filter[];
+}, fields: FieldSchema[]) => Filter[];

package/dist/src/helpers.js

@@ -0,0 +1,28 @@
+/**
+ * Use this helper function to retrieve the applicable filters from the context object. While using filters
+ * directly from context might work, it doesn't offer any guarantees about the shape of the filters nor the
+ * validity of the fields against which the filters are applied. On the other hand, this function ensures that
+ * all filters are valid and that the fields against which the filters are applied are present in the schema.
+ *
+ * @param context The object containing the raw filters
+ * @param fields The schema of the item against which the filters are applied
+ * @returns The validated filters
+ */
+export const getApplicableFilters = (context, fields) => {
+    const applicableFilters = [];
+    for (const filter of context.filters) {
+        let field = undefined;
+        const filterFieldParts = filter.field.split(':', 2);
+        switch (filterFieldParts[0]) {
+            case 'semantic':
+                field = fields.find(f => f.semantic === filterFieldParts[1]);
+                break;
+            default:
+                field = fields.find(f => f.name === filterFieldParts[0]);
+        }
+        if (field) {
+            applicableFilters.push({ ...filter, field: field.name });
+        }
+    }
+    return applicableFilters;
+};

package/dist/src/index.cjs

@@ -353,13 +353,13 @@ function buildHttpError(responseStatus, message) {
     return httpError;
 }
 
-const middleware$8 = (req, res, next) => {
+const middleware$9 = (req, res, next) => {
     res.locals.correlationId = req.header('X-Unito-Correlation-Id') ?? uuid__namespace.v4();
     next();
 };
 
 const ADDITIONAL_CONTEXT_HEADER = 'X-Unito-Additional-Logging-Context';
-const middleware$7 = (req, res, next) => {
+const middleware$8 = (req, res, next) => {
     const logger = new Logger({ correlation_id: res.locals.correlationId });
     res.locals.logger = logger;
     const rawAdditionalContext = req.header(ADDITIONAL_CONTEXT_HEADER);
@@ -376,7 +376,7 @@ const middleware$7 = (req, res, next) => {
 };
 
 const CREDENTIALS_HEADER = 'X-Unito-Credentials';
-const middleware$6 = (req, res, next) => {
+const middleware$7 = (req, res, next) => {
     const credentialsHeader = req.header(CREDENTIALS_HEADER);
     if (credentialsHeader) {
         let credentials;
@@ -392,7 +392,7 @@ const middleware$6 = (req, res, next) => {
 };
 
 const OPERATION_DEADLINE_HEADER = 'X-Unito-Operation-Deadline';
-const middleware$5 = (req, res, next) => {
+const middleware$6 = (req, res, next) => {
     const operationDeadlineHeader = Number(req.header(OPERATION_DEADLINE_HEADER));
     if (operationDeadlineHeader) {
         // `operationDeadlineHeader` represents a timestamp in the future, in seconds.
@@ -413,7 +413,7 @@ const middleware$5 = (req, res, next) => {
 };
 
 const SECRETS_HEADER = 'X-Unito-Secrets';
-const middleware$4 = (req, res, next) => {
+const middleware$5 = (req, res, next) => {
     const secretsHeader = req.header(SECRETS_HEADER);
     if (secretsHeader) {
         let secrets;
@@ -428,7 +428,7 @@ const middleware$4 = (req, res, next) => {
     next();
 };
 
-const middleware$3 = (req, res, next) => {
+const middleware$4 = (req, res, next) => {
     const rawSelect = req.query.select;
     if (typeof rawSelect === 'string') {
         res.locals.selects = rawSelect.split(',');
@@ -439,7 +439,7 @@ const middleware$3 = (req, res, next) => {
     next();
 };
 
-const middleware$2 = (err, _req, res, next) => {
+const middleware$3 = (err, _req, res, next) => {
     if (res.headersSent) {
         return next(err);
     }
@@ -475,12 +475,15 @@ const middleware$2 = (err, _req, res, next) => {
     res.status(Number(error.code)).json(error);
 };
 
-const middleware$1 = (req, res, next) => {
+const middleware$2 = (req, res, next) => {
     if (req.originalUrl !== '/health') {
         res.on('finish', function () {
             const error = res.locals.error;
-            const message = `${req.method} ${req.originalUrl} ${res.statusCode}`;
+            const durationInNs = Number(process.hrtime.bigint() - res.locals.requestStartTime);
+            const durationInMs = (durationInNs / 1_000_000) | 0;
+            const message = `${req.method} ${req.originalUrl} ${res.statusCode} - ${durationInMs} ms`;
             const metadata = {
+                duration: durationInNs,
                 // Use reserved and standard attributes of Datadog
                 // https://app.datadoghq.com/logs/pipelines/standard-attributes
                 http: { method: req.method, status_code: res.statusCode, url_details: { path: req.originalUrl } },
@@ -508,7 +511,7 @@ const middleware$1 = (req, res, next) => {
     next();
 };
 
-const middleware = (req, res, _next) => {
+const middleware$1 = (req, res, _next) => {
     const error = {
         code: '404',
         message: `Path ${req.path} not found.`,
@@ -516,6 +519,11 @@ const middleware = (req, res, _next) => {
     res.status(404).json(error);
 };
 
+const middleware = (_, res, next) => {
+    res.locals.requestStartTime = process.hrtime.bigint();
+    next();
+};
+
 function assertValidPath(path) {
     if (!path.startsWith('/')) {
         throw new InvalidHandler(`The provided path '${path}' is invalid. All paths must start with a '/'.`);
@@ -903,13 +911,14 @@ class Integration {
         app.set('query parser', 'extended');
         app.use(express.json());
         // Must be one of the first handlers (to catch all the errors).
-        app.use(middleware$1);
+        app.use(middleware$2);
+        app.use(middleware);
+        app.use(middleware$9);
         app.use(middleware$8);
         app.use(middleware$7);
-        app.use(middleware$6);
-        app.use(middleware$4);
-        app.use(middleware$3);
         app.use(middleware$5);
+        app.use(middleware$4);
+        app.use(middleware$6);
         // Load handlers as needed.
         if (this.handlers.length) {
             for (const handler of this.handlers) {
@@ -925,9 +934,9 @@ class Integration {
             process.exit(1);
         }
         // Must be the (last - 1) handler.
-        app.use(middleware$2);
+        app.use(middleware$3);
         // Must be the last handler.
-        app.use(middleware);
+        app.use(middleware$1);
         // Start the server.
         this.instance = app.listen(this.port, () => console.info(`Server started on port ${this.port}.`));
     }
@@ -976,6 +985,26 @@ class Provider {
             },
         });
     }
+    /**
+     * Performs a GET request to the provider and return the response as a ReadableStream.
+     *
+     * Uses the prepareRequest function to get the base URL and any specific headers to add to the request and by default
+     * adds the following headers:
+     * - Accept: application/octet-stream
+     *
+     * @param endpoint Path to the provider's resource. Will be added to the URL returned by the prepareRequest function.
+     * @param options RequestOptions used to adjust the call made to the provider (e.g. used to override default headers).
+     * @returns The streaming {@link Response} extracted from the provider.
+     */
+    async streamingGet(endpoint, options) {
+        return this.fetchWrapper(endpoint, null, {
+            ...options,
+            method: 'GET',
+            defaultHeaders: {
+                Accept: 'application/octet-stream',
+            },
+        });
+    }
     /**
      * Performs a POST request to the provider.
      *
@@ -1103,21 +1132,68 @@ class Provider {
                 const textResult = await response.text();
                 throw buildHttpError(response.status, textResult);
             }
-            try {
-                const body = response.body ? await response.json() : undefined;
-                return { status: response.status, headers: response.headers, body };
+            const responseContentType = response.headers.get('content-type');
+            let body;
+            if (headers.Accept === 'application/json') {
+                // Validate that the response content type is at least similar to what we expect
+                // (Provider's response Content-Type might be more specific, e.g. application/json;charset=utf-8)
+                // Default to application/json if no Content-Type header is provided
+                if (responseContentType && !responseContentType.includes('application/json')) {
+                    throw buildHttpError(500, `Unsupported content-type. Expected 'application/json', got '${responseContentType}'`);
+                }
+                try {
+                    body = response.body ? await response.json() : undefined;
+                }
+                catch (err) {
+                    throw buildHttpError(500, `Invalid JSON response`);
+                }
             }
-            catch {
-                throw buildHttpError(400, 'Invalid JSON response');
+            else if (headers.Accept == 'application/octet-stream') {
+                // When we expect octet-stream, we accept any Content-Type the provider sends us, we just want to stream it.
+                body = response.body;
             }
+            else {
+                throw buildHttpError(500, 'Unsupported accept header');
+            }
+            return { status: response.status, headers: response.headers, body };
         };
         return this.rateLimiter ? this.rateLimiter(options, callToProvider) : callToProvider();
     }
 }
 
+/**
+ * Use this helper function to retrieve the applicable filters from the context object. While using filters
+ * directly from context might work, it doesn't offer any guarantees about the shape of the filters nor the
+ * validity of the fields against which the filters are applied. On the other hand, this function ensures that
+ * all filters are valid and that the fields against which the filters are applied are present in the schema.
+ *
+ * @param context The object containing the raw filters
+ * @param fields The schema of the item against which the filters are applied
+ * @returns The validated filters
+ */
+const getApplicableFilters = (context, fields) => {
+    const applicableFilters = [];
+    for (const filter of context.filters) {
+        let field = undefined;
+        const filterFieldParts = filter.field.split(':', 2);
+        switch (filterFieldParts[0]) {
+            case 'semantic':
+                field = fields.find(f => f.semantic === filterFieldParts[1]);
+                break;
+            default:
+                field = fields.find(f => f.name === filterFieldParts[0]);
+        }
+        if (field) {
+            applicableFilters.push({ ...filter, field: field.name });
+        }
+    }
+    return applicableFilters;
+};
+
 exports.Api = integrationApi__namespace;
 exports.Cache = Cache;
 exports.Handler = Handler;
 exports.HttpErrors = httpErrors;
 exports.Integration = Integration;
 exports.Provider = Provider;
+exports.getApplicableFilters = getApplicableFilters;

package/dist/src/index.d.ts

@@ -5,6 +5,8 @@ export * from './handler.js';
 export { Provider, type Response as ProviderResponse, type RequestOptions as ProviderRequestOptions, type RateLimiter, } from './resources/provider.js';
 export type { Secrets } from './middlewares/secrets.js';
 export type { Credentials } from './middlewares/credentials.js';
+export type { Filter } from './middlewares/filters.js';
 export * as HttpErrors from './httpErrors.js';
+export { getApplicableFilters } from './helpers.js';
 export * from './resources/context.js';
 export { type default as Logger } from './resources/logger.js';

package/dist/src/index.js

@@ -5,5 +5,6 @@ export { default as Integration } from './integration.js';
 export * from './handler.js';
 export { Provider, } from './resources/provider.js';
 export * as HttpErrors from './httpErrors.js';
+export { getApplicableFilters } from './helpers.js';
 export * from './resources/context.js';
 /* c8 ignore stop */

package/dist/src/integration.js

@@ -9,6 +9,7 @@ import selectsMiddleware from './middlewares/selects.js';
 import errorsMiddleware from './middlewares/errors.js';
 import finishMiddleware from './middlewares/finish.js';
 import notFoundMiddleware from './middlewares/notFound.js';
+import requestStartTimeMiddleware from './middlewares/requestStartTime.js';
 import { Handler } from './handler.js';
 function printErrorMessage(message) {
     console.error();
@@ -112,6 +113,7 @@ export default class Integration {
         app.use(express.json());
         // Must be one of the first handlers (to catch all the errors).
         app.use(finishMiddleware);
+        app.use(requestStartTimeMiddleware);
         app.use(correlationIdMiddleware);
         app.use(loggerMiddleware);
         app.use(credentialsMiddleware);

package/dist/src/middlewares/finish.d.ts

@@ -6,6 +6,7 @@ declare global {
         interface Locals {
             logger: Logger;
             error?: ApiError;
+            requestStartTime: bigint;
         }
     }
 }

package/dist/src/middlewares/finish.js

@@ -2,8 +2,11 @@ const middleware = (req, res, next) => {
     if (req.originalUrl !== '/health') {
         res.on('finish', function () {
             const error = res.locals.error;
-            const message = `${req.method} ${req.originalUrl} ${res.statusCode}`;
+            const durationInNs = Number(process.hrtime.bigint() - res.locals.requestStartTime);
+            const durationInMs = (durationInNs / 1_000_000) | 0;
+            const message = `${req.method} ${req.originalUrl} ${res.statusCode} - ${durationInMs} ms`;
             const metadata = {
+                duration: durationInNs,
                 // Use reserved and standard attributes of Datadog
                 // https://app.datadoghq.com/logs/pipelines/standard-attributes
                 http: { method: req.method, status_code: res.statusCode, url_details: { path: req.originalUrl } },

package/dist/src/middlewares/logger.d.ts

@@ -3,8 +3,8 @@ import Logger from '../resources/logger.js';
 declare global {
     namespace Express {
         interface Locals {
-            correlationId: string;
             logger: Logger;
+            correlationId: string;
         }
     }
 }

package/dist/src/middlewares/requestStartTime.d.ts

@@ -0,0 +1,10 @@
+import { Request, Response, NextFunction } from 'express';
+declare global {
+    namespace Express {
+        interface Locals {
+            requestStartTime: bigint;
+        }
+    }
+}
+declare const middleware: (_: Request, res: Response, next: NextFunction) => void;
+export default middleware;

package/dist/src/middlewares/requestStartTime.js

@@ -0,0 +1,5 @@
+const middleware = (_, res, next) => {
+    res.locals.requestStartTime = process.hrtime.bigint();
+    next();
+};
+export default middleware;

package/dist/src/resources/provider.d.ts

@@ -93,6 +93,18 @@ export declare class Provider {
      * @returns The {@link Response} extracted from the provider.
      */
     get<T>(endpoint: string, options: RequestOptions): Promise<Response<T>>;
+    /**
+     * Performs a GET request to the provider and return the response as a ReadableStream.
+     *
+     * Uses the prepareRequest function to get the base URL and any specific headers to add to the request and by default
+     * adds the following headers:
+     * - Accept: application/octet-stream
+     *
+     * @param endpoint Path to the provider's resource. Will be added to the URL returned by the prepareRequest function.
+     * @param options RequestOptions used to adjust the call made to the provider (e.g. used to override default headers).
+     * @returns The streaming {@link Response} extracted from the provider.
+     */
+    streamingGet(endpoint: string, options: RequestOptions): Promise<Response<ReadableStream<Uint8Array>>>;
     /**
      * Performs a POST request to the provider.
      *

package/dist/src/resources/provider.js

@@ -42,6 +42,26 @@ export class Provider {
             },
         });
     }
+    /**
+     * Performs a GET request to the provider and return the response as a ReadableStream.
+     *
+     * Uses the prepareRequest function to get the base URL and any specific headers to add to the request and by default
+     * adds the following headers:
+     * - Accept: application/octet-stream
+     *
+     * @param endpoint Path to the provider's resource. Will be added to the URL returned by the prepareRequest function.
+     * @param options RequestOptions used to adjust the call made to the provider (e.g. used to override default headers).
+     * @returns The streaming {@link Response} extracted from the provider.
+     */
+    async streamingGet(endpoint, options) {
+        return this.fetchWrapper(endpoint, null, {
+            ...options,
+            method: 'GET',
+            defaultHeaders: {
+                Accept: 'application/octet-stream',
+            },
+        });
+    }
     /**
      * Performs a POST request to the provider.
      *
@@ -169,13 +189,30 @@ export class Provider {
                 const textResult = await response.text();
                 throw buildHttpError(response.status, textResult);
             }
-            try {
-                const body = response.body ? await response.json() : undefined;
-                return { status: response.status, headers: response.headers, body };
+            const responseContentType = response.headers.get('content-type');
+            let body;
+            if (headers.Accept === 'application/json') {
+                // Validate that the response content type is at least similar to what we expect
+                // (Provider's response Content-Type might be more specific, e.g. application/json;charset=utf-8)
+                // Default to application/json if no Content-Type header is provided
+                if (responseContentType && !responseContentType.includes('application/json')) {
+                    throw buildHttpError(500, `Unsupported content-type. Expected 'application/json', got '${responseContentType}'`);
+                }
+                try {
+                    body = response.body ? await response.json() : undefined;
+                }
+                catch (err) {
+                    throw buildHttpError(500, `Invalid JSON response`);
+                }
+            }
+            else if (headers.Accept == 'application/octet-stream') {
+                // When we expect octet-stream, we accept any Content-Type the provider sends us, we just want to stream it.
+                body = response.body;
             }
-            catch {
-                throw buildHttpError(400, 'Invalid JSON response');
+            else {
+                throw buildHttpError(500, 'Unsupported accept header');
             }
+            return { status: response.status, headers: response.headers, body };
         };
         return this.rateLimiter ? this.rateLimiter(options, callToProvider) : callToProvider();
     }

package/dist/test/handler.test.js

@@ -77,7 +77,7 @@ describe('Handler', () => {
                     credentials: { foo: 'bar' },
                 },
             };
-            await handler(request, response);
+            await handler(request, response, () => { });
             return { body, statusCode };
         }
         it('returns a router', async () => {

package/dist/test/helpers.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/test/helpers.test.js

@@ -0,0 +1,57 @@
+import { FieldValueType, OperatorType, Semantic } from '@unito/integration-api';
+import assert from 'node:assert/strict';
+import { describe, it } from 'node:test';
+import { getApplicableFilters } from '../src/helpers.js';
+describe('Helpers', () => {
+    describe('getApplicableFilters', () => {
+        it('returns only filters for defined fields', () => {
+            const actual = getApplicableFilters({
+                filters: [
+                    { field: 'status', operator: OperatorType.EQUAL, values: ['active', 'pending'] },
+                    { field: 'email', operator: OperatorType.IS_NOT_NULL, values: [] },
+                ],
+            }, [
+                {
+                    name: 'status',
+                    label: 'Status',
+                    type: FieldValueType.STRING,
+                },
+            ]);
+            const expected = [{ field: 'status', operator: OperatorType.EQUAL, values: ['active', 'pending'] }];
+            assert.deepEqual(actual, expected);
+        });
+        it('translates semantics into field names', () => {
+            const actual = getApplicableFilters({
+                filters: [
+                    { field: 'semantic:displayName', operator: OperatorType.START_WITH, values: ['Bob'] },
+                    { field: 'semantic:createdAt', operator: OperatorType.EQUAL, values: ['2021-01-01'] },
+                ],
+            }, [
+                {
+                    name: 'name',
+                    label: 'Name',
+                    type: FieldValueType.STRING,
+                    semantic: Semantic.DISPLAY_NAME,
+                },
+            ]);
+            const expected = [{ field: 'name', operator: OperatorType.START_WITH, values: ['Bob'] }];
+            assert.deepEqual(actual, expected);
+        });
+        it('gracefully handle garbage', () => {
+            const actual = getApplicableFilters({
+                filters: [
+                    { field: '...', operator: OperatorType.EQUAL, values: [] },
+                    { field: ':', operator: OperatorType.EQUAL, values: [] },
+                    { field: '', operator: OperatorType.EQUAL, values: [] },
+                ],
+            }, [
+                {
+                    name: 'status',
+                    label: 'Status',
+                    type: FieldValueType.STRING,
+                },
+            ]);
+            assert.deepEqual(actual, []);
+        });
+    });
+});

package/dist/test/middlewares/finish.test.js

@@ -11,6 +11,7 @@ describe('finish middleware', () => {
                 eventHandler = func;
             },
             locals: {
+                requestStartTime: 0n,
                 logger: {
                     info: (_message) => {
                         expected = 'works!';
@@ -32,6 +33,7 @@ describe('finish middleware', () => {
                 eventHandler = func;
             },
             locals: {
+                requestStartTime: 0n,
                 logger: {
                     error: (_message) => {
                         expected = 'works!';
@@ -53,6 +55,7 @@ describe('finish middleware', () => {
                 eventHandler = func;
             },
             locals: {
+                requestStartTime: 0n,
                 logger: {
                     info: (_message) => {
                         expected = 'ohoh!';

package/dist/test/resources/provider.test.js

@@ -222,9 +222,52 @@ describe('Provider', () => {
         ]);
         assert.deepEqual(actualResponse, { status: 204, headers: response.headers, body: undefined });
     });
+    it('returns valid json response', async (context) => {
+        const response = new Response(`{ "validJson": true }`, {
+            status: 200,
+            headers: { 'Content-Type': 'application/json;charset=utf-8' },
+        });
+        context.mock.method(global, 'fetch', () => Promise.resolve(response));
+        const providerResponse = await provider.get('/endpoint/123', {
+            credentials: { apiKey: 'apikey#1111' },
+            logger: logger,
+            signal: new AbortController().signal,
+        });
+        assert.ok(providerResponse);
+        assert.ok(providerResponse.body);
+        assert.equal(providerResponse.body.validJson, true);
+    });
+    it('returns successfully on missing Content-Type header', async (context) => {
+        const response = new Response(undefined, {
+            status: 201,
+        });
+        context.mock.method(global, 'fetch', () => Promise.resolve(response));
+        const providerResponse = await provider.get('/endpoint/123', {
+            credentials: { apiKey: 'apikey#1111' },
+            logger: logger,
+            signal: new AbortController().signal,
+        });
+        assert.ok(providerResponse);
+        assert.equal(providerResponse.body, undefined);
+    });
+    it('returns streamable response on streaming get calls', async (context) => {
+        const response = new Response(`IMAGINE A HUGE PAYLOAD`, {
+            status: 200,
+            headers: { 'Content-Type': 'video/mp4' },
+        });
+        context.mock.method(global, 'fetch', () => Promise.resolve(response));
+        const providerResponse = await provider.streamingGet('/endpoint/123', {
+            credentials: { apiKey: 'apikey#1111' },
+            logger: logger,
+            signal: new AbortController().signal,
+        });
+        assert.ok(providerResponse);
+        assert.ok(providerResponse.body instanceof ReadableStream);
+    });
     it('throws on invalid json response', async (context) => {
         const response = new Response('{invalidJSON}', {
             status: 200,
+            headers: { 'Content-Type': 'application/json' },
         });
         context.mock.method(global, 'fetch', () => Promise.resolve(response));
         let error;
@@ -238,9 +281,29 @@ describe('Provider', () => {
         catch (e) {
             error = e;
         }
-        assert.ok(error instanceof HttpErrors.BadRequestError);
+        assert.ok(error instanceof HttpErrors.HttpError);
         assert.equal(error.message, 'Invalid JSON response');
     });
+    it('throws on unexpected content-type response', async (context) => {
+        const response = new Response('text', {
+            status: 200,
+            headers: { 'Content-Type': 'application/text' },
+        });
+        context.mock.method(global, 'fetch', () => Promise.resolve(response));
+        let error;
+        try {
+            await provider.get('/endpoint/123', {
+                credentials: { apiKey: 'apikey#1111' },
+                logger: logger,
+                signal: new AbortController().signal,
+            });
+        }
+        catch (e) {
+            error = e;
+        }
+        assert.ok(error instanceof HttpErrors.HttpError);
+        assert.equal(error.message, `Unsupported content-type. Expected 'application/json', got 'application/text'`);
+    });
     it('throws on status 400', async (context) => {
         const response = new Response('response body', {
             status: 400,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@unito/integration-sdk",
-  "version": "1.0.4",
+  "version": "1.0.6",
   "description": "Integration SDK",
   "type": "module",
   "types": "dist/src/index.d.ts",

package/src/helpers.ts

@@ -0,0 +1,37 @@
+import { FieldSchema } from '@unito/integration-api';
+
+import { Filter } from './index.js';
+
+/**
+ * Use this helper function to retrieve the applicable filters from the context object. While using filters
+ * directly from context might work, it doesn't offer any guarantees about the shape of the filters nor the
+ * validity of the fields against which the filters are applied. On the other hand, this function ensures that
+ * all filters are valid and that the fields against which the filters are applied are present in the schema.
+ *
+ * @param context The object containing the raw filters
+ * @param fields The schema of the item against which the filters are applied
+ * @returns The validated filters
+ */
+export const getApplicableFilters = (context: { filters: Filter[] }, fields: FieldSchema[]): Filter[] => {
+  const applicableFilters: Filter[] = [];
+
+  for (const filter of context.filters) {
+    let field: FieldSchema | undefined = undefined;
+
+    const filterFieldParts = filter.field.split(':', 2);
+
+    switch (filterFieldParts[0]) {
+      case 'semantic':
+        field = fields.find(f => f.semantic === filterFieldParts[1]);
+        break;
+      default:
+        field = fields.find(f => f.name === filterFieldParts[0]);
+    }
+
+    if (field) {
+      applicableFilters.push({ ...filter, field: field.name });
+    }
+  }
+
+  return applicableFilters;
+};

package/src/index.ts

@@ -11,7 +11,9 @@ export {
 } from './resources/provider.js';
 export type { Secrets } from './middlewares/secrets.js';
 export type { Credentials } from './middlewares/credentials.js';
+export type { Filter } from './middlewares/filters.js';
 export * as HttpErrors from './httpErrors.js';
+export { getApplicableFilters } from './helpers.js';
 export * from './resources/context.js';
 export { type default as Logger } from './resources/logger.js';
 /* c8 ignore stop */

package/src/integration.ts

@@ -11,6 +11,7 @@ import selectsMiddleware from './middlewares/selects.js';
 import errorsMiddleware from './middlewares/errors.js';
 import finishMiddleware from './middlewares/finish.js';
 import notFoundMiddleware from './middlewares/notFound.js';
+import requestStartTimeMiddleware from './middlewares/requestStartTime.js';
 import { HandlersInput, Handler } from './handler.js';
 
 function printErrorMessage(message: string) {
@@ -128,6 +129,7 @@ export default class Integration {
     // Must be one of the first handlers (to catch all the errors).
     app.use(finishMiddleware);
 
+    app.use(requestStartTimeMiddleware);
     app.use(correlationIdMiddleware);
     app.use(loggerMiddleware);
     app.use(credentialsMiddleware);

package/src/middlewares/finish.ts

@@ -8,6 +8,7 @@ declare global {
     interface Locals {
       logger: Logger;
       error?: ApiError;
+      requestStartTime: bigint;
     }
   }
 }
@@ -17,8 +18,12 @@ const middleware = (req: Request, res: Response, next: NextFunction) => {
     res.on('finish', function () {
       const error = res.locals.error;
 
-      const message = `${req.method} ${req.originalUrl} ${res.statusCode}`;
+      const durationInNs = Number(process.hrtime.bigint() - res.locals.requestStartTime);
+      const durationInMs = (durationInNs / 1_000_000) | 0;
+
+      const message = `${req.method} ${req.originalUrl} ${res.statusCode} - ${durationInMs} ms`;
       const metadata: Metadata = {
+        duration: durationInNs,
         // Use reserved and standard attributes of Datadog
         // https://app.datadoghq.com/logs/pipelines/standard-attributes
         http: { method: req.method, status_code: res.statusCode, url_details: { path: req.originalUrl } },

package/src/middlewares/logger.ts

@@ -5,8 +5,8 @@ declare global {
   // eslint-disable-next-line @typescript-eslint/no-namespace
   namespace Express {
     interface Locals {
-      correlationId: string;
       logger: Logger;
+      correlationId: string;
     }
   }
 }

package/src/middlewares/requestStartTime.ts

@@ -0,0 +1,18 @@
+import { Request, Response, NextFunction } from 'express';
+
+declare global {
+  // eslint-disable-next-line @typescript-eslint/no-namespace
+  namespace Express {
+    interface Locals {
+      requestStartTime: bigint;
+    }
+  }
+}
+
+const middleware = (_: Request, res: Response, next: NextFunction) => {
+  res.locals.requestStartTime = process.hrtime.bigint();
+
+  next();
+};
+
+export default middleware;

package/src/resources/provider.ts

@@ -102,6 +102,27 @@ export class Provider {
     });
   }
 
+  /**
+   * Performs a GET request to the provider and return the response as a ReadableStream.
+   *
+   * Uses the prepareRequest function to get the base URL and any specific headers to add to the request and by default
+   * adds the following headers:
+   * - Accept: application/octet-stream
+   *
+   * @param endpoint Path to the provider's resource. Will be added to the URL returned by the prepareRequest function.
+   * @param options RequestOptions used to adjust the call made to the provider (e.g. used to override default headers).
+   * @returns The streaming {@link Response} extracted from the provider.
+   */
+  public async streamingGet(endpoint: string, options: RequestOptions): Promise<Response<ReadableStream<Uint8Array>>> {
+    return this.fetchWrapper<ReadableStream<Uint8Array>>(endpoint, null, {
+      ...options,
+      method: 'GET',
+      defaultHeaders: {
+        Accept: 'application/octet-stream',
+      },
+    });
+  }
+
   /**
    * Performs a POST request to the provider.
    *
@@ -249,12 +270,33 @@ export class Provider {
         throw buildHttpError(response.status, textResult);
       }
 
-      try {
-        const body: T = response.body ? await response.json() : undefined;
-        return { status: response.status, headers: response.headers, body };
-      } catch {
-        throw buildHttpError(400, 'Invalid JSON response');
+      const responseContentType = response.headers.get('content-type');
+      let body: T;
+
+      if (headers.Accept === 'application/json') {
+        // Validate that the response content type is at least similar to what we expect
+        // (Provider's response Content-Type might be more specific, e.g. application/json;charset=utf-8)
+        // Default to application/json if no Content-Type header is provided
+        if (responseContentType && !responseContentType.includes('application/json')) {
+          throw buildHttpError(
+            500,
+            `Unsupported content-type. Expected 'application/json', got '${responseContentType}'`,
+          );
+        }
+
+        try {
+          body = response.body ? await response.json() : undefined;
+        } catch (err) {
+          throw buildHttpError(500, `Invalid JSON response`);
+        }
+      } else if (headers.Accept == 'application/octet-stream') {
+        // When we expect octet-stream, we accept any Content-Type the provider sends us, we just want to stream it.
+        body = response.body as T;
+      } else {
+        throw buildHttpError(500, 'Unsupported accept header');
       }
+
+      return { status: response.status, headers: response.headers, body };
     };
 
     return this.rateLimiter ? this.rateLimiter(options, callToProvider) : callToProvider();

package/test/handler.test.ts

@@ -1,4 +1,4 @@
-import { Request, Response } from 'express';
+import { Request, Response, NextFunction } from 'express';
 import assert from 'node:assert/strict';
 import { afterEach, beforeEach, describe, it, mock } from 'node:test';
 import {
@@ -80,7 +80,7 @@ describe('Handler', () => {
 
   describe('generate', () => {
     async function executeHandler(
-      handler: (req: Request, res: Response) => Promise<void>,
+      handler: (req: Request, res: Response, callback: NextFunction) => Promise<void>,
       request: Record<string, unknown> = {},
     ) {
       let body: Record<string, unknown> = {};
@@ -99,7 +99,7 @@ describe('Handler', () => {
         },
       } as unknown as Response;
 
-      await handler(request as unknown as Request, response);
+      await handler(request as unknown as Request, response, () => {});
 
       return { body, statusCode };
     }
@@ -117,43 +117,43 @@ describe('Handler', () => {
       const routes = router.stack;
 
       assert.equal(routes.length, 5);
-      assert.equal(routes[0].route.path, '/foo');
-      assert.equal(routes[0].route.methods.get, true);
-      assert.equal(routes[1].route.path, '/foo');
-      assert.equal(routes[1].route.methods.post, true);
-      assert.equal(routes[2].route.path, '/foo/:bar');
-      assert.equal(routes[2].route.methods.get, true);
-      assert.equal(routes[3].route.path, '/foo/:bar');
-      assert.equal(routes[3].route.methods.patch, true);
-      assert.equal(routes[4].route.path, '/foo/:bar');
-      assert.equal(routes[4].route.methods.delete, true);
+      assert.equal(routes[0]!.route!.path, '/foo');
+      assert.equal((routes[0]!.route! as any).methods.get, true);
+      assert.equal(routes[1]!.route!.path, '/foo');
+      assert.equal((routes[1]!.route! as any).methods.post, true);
+      assert.equal(routes[2]!.route!.path, '/foo/:bar');
+      assert.equal((routes[2]!.route! as any).methods.get, true);
+      assert.equal(routes[3]!.route!.path, '/foo/:bar');
+      assert.equal((routes[3]!.route! as any).methods.patch, true);
+      assert.equal(routes[4]!.route!.path, '/foo/:bar');
+      assert.equal((routes[4]!.route! as any).methods.delete, true);
 
       // GetCollection.
-      assert.deepEqual(await executeHandler(routes[0].route.stack[0].handle), {
+      assert.deepEqual(await executeHandler(routes[0]!.route!.stack[0]!.handle), {
         body: { info: {}, data: [] },
         statusCode: 200,
       });
 
       // CreateItem.
-      assert.deepEqual(await executeHandler(routes[1].route.stack[0].handle, { body: { foo: 'bar' } }), {
+      assert.deepEqual(await executeHandler(routes[1]!.route!.stack[0]!.handle, { body: { foo: 'bar' } }), {
         body: { fields: {}, path: '/' },
         statusCode: 201,
       });
 
       // GetItem.
-      assert.deepEqual(await executeHandler(routes[2].route.stack[0].handle), {
+      assert.deepEqual(await executeHandler(routes[2]!.route!.stack[0]!.handle), {
         body: { fields: {}, relations: [] },
         statusCode: 200,
       });
 
       // UpdateItem.
-      assert.deepEqual(await executeHandler(routes[3].route.stack[0].handle, { body: { foo: 'bar' } }), {
+      assert.deepEqual(await executeHandler(routes[3]!.route!.stack[0]!.handle, { body: { foo: 'bar' } }), {
         body: { fields: {}, relations: [] },
         statusCode: 200,
       });
 
       // DeleteItem.
-      assert.deepEqual(await executeHandler(routes[4].route.stack[0].handle), {
+      assert.deepEqual(await executeHandler(routes[4]!.route!.stack[0]!.handle), {
         body: null,
         statusCode: 204,
       });
@@ -168,11 +168,11 @@ describe('Handler', () => {
       const routes = router.stack;
 
       assert.equal(routes.length, 1);
-      assert.equal(routes[0].route.path, '/foo/:bar/baz');
-      assert.equal(routes[0].route.methods.get, true);
+      assert.equal(routes[0]!.route!.path, '/foo/:bar/baz');
+      assert.equal((routes[0]!.route as any).methods.get, true);
 
       // GetCollection.
-      assert.deepEqual(await executeHandler(routes[0].route.stack[0].handle), {
+      assert.deepEqual(await executeHandler(routes[0]!.route!.stack[0]!.handle), {
         body: { info: {}, data: [] },
         statusCode: 200,
       });
@@ -197,13 +197,13 @@ describe('Handler', () => {
       const routes = router.stack;
 
       // CreateItemRequestPayload.
-      const createHandler = routes[0].route.stack[0].handle;
+      const createHandler = routes[0]!.route!.stack[0]!.handle;
       await assert.doesNotReject(async () => await executeHandler(createHandler, { body: { foo: 'bar' } }));
       await assert.rejects(async () => await executeHandler(createHandler, { body: null }), BadRequestError);
       await assert.rejects(async () => await executeHandler(createHandler, { body: 'not json' }), BadRequestError);
 
       // UpdateItemRequestPayload.
-      const updateHandler = routes[0].route.stack[0].handle;
+      const updateHandler = routes[0]!.route!.stack[0]!.handle;
       await assert.doesNotReject(async () => await executeHandler(updateHandler, { body: { foo: 'bar' } }));
       await assert.rejects(async () => await executeHandler(updateHandler, { body: null }), BadRequestError);
       await assert.rejects(async () => await executeHandler(updateHandler, { body: 'not json' }), BadRequestError);

package/test/helpers.test.ts

@@ -0,0 +1,75 @@
+import { FieldValueType, OperatorType, Semantic } from '@unito/integration-api';
+
+import assert from 'node:assert/strict';
+import { describe, it } from 'node:test';
+import { getApplicableFilters } from '../src/helpers.js';
+
+describe('Helpers', () => {
+  describe('getApplicableFilters', () => {
+    it('returns only filters for defined fields', () => {
+      const actual = getApplicableFilters(
+        {
+          filters: [
+            { field: 'status', operator: OperatorType.EQUAL, values: ['active', 'pending'] },
+            { field: 'email', operator: OperatorType.IS_NOT_NULL, values: [] },
+          ],
+        },
+        [
+          {
+            name: 'status',
+            label: 'Status',
+            type: FieldValueType.STRING,
+          },
+        ],
+      );
+
+      const expected = [{ field: 'status', operator: OperatorType.EQUAL, values: ['active', 'pending'] }];
+
+      assert.deepEqual(actual, expected);
+    });
+
+    it('translates semantics into field names', () => {
+      const actual = getApplicableFilters(
+        {
+          filters: [
+            { field: 'semantic:displayName', operator: OperatorType.START_WITH, values: ['Bob'] },
+            { field: 'semantic:createdAt', operator: OperatorType.EQUAL, values: ['2021-01-01'] },
+          ],
+        },
+        [
+          {
+            name: 'name',
+            label: 'Name',
+            type: FieldValueType.STRING,
+            semantic: Semantic.DISPLAY_NAME,
+          },
+        ],
+      );
+
+      const expected = [{ field: 'name', operator: OperatorType.START_WITH, values: ['Bob'] }];
+
+      assert.deepEqual(actual, expected);
+    });
+
+    it('gracefully handle garbage', () => {
+      const actual = getApplicableFilters(
+        {
+          filters: [
+            { field: '...', operator: OperatorType.EQUAL, values: [] },
+            { field: ':', operator: OperatorType.EQUAL, values: [] },
+            { field: '', operator: OperatorType.EQUAL, values: [] },
+          ],
+        },
+        [
+          {
+            name: 'status',
+            label: 'Status',
+            type: FieldValueType.STRING,
+          },
+        ],
+      );
+
+      assert.deepEqual(actual, []);
+    });
+  });
+});

package/test/middlewares/finish.test.ts

@@ -14,6 +14,7 @@ describe('finish middleware', () => {
         eventHandler = func;
       },
       locals: {
+        requestStartTime: 0n,
         logger: {
           info: (_message: string) => {
             expected = 'works!';
@@ -39,6 +40,7 @@ describe('finish middleware', () => {
         eventHandler = func;
       },
       locals: {
+        requestStartTime: 0n,
         logger: {
           error: (_message: string) => {
             expected = 'works!';
@@ -64,6 +66,7 @@ describe('finish middleware', () => {
         eventHandler = func;
       },
       locals: {
+        requestStartTime: 0n,
         logger: {
           info: (_message: string) => {
             expected = 'ohoh!';

package/test/resources/provider.test.ts

@@ -266,9 +266,64 @@ describe('Provider', () => {
     assert.deepEqual(actualResponse, { status: 204, headers: response.headers, body: undefined });
   });
 
+  it('returns valid json response', async context => {
+    const response = new Response(`{ "validJson": true }`, {
+      status: 200,
+      headers: { 'Content-Type': 'application/json;charset=utf-8' },
+    });
+
+    context.mock.method(global, 'fetch', () => Promise.resolve(response));
+
+    const providerResponse = await provider.get<{ validJson: boolean }>('/endpoint/123', {
+      credentials: { apiKey: 'apikey#1111' },
+      logger: logger,
+      signal: new AbortController().signal,
+    });
+
+    assert.ok(providerResponse);
+    assert.ok(providerResponse.body);
+    assert.equal(providerResponse.body.validJson, true);
+  });
+
+  it('returns successfully on missing Content-Type header', async context => {
+    const response = new Response(undefined, {
+      status: 201,
+    });
+
+    context.mock.method(global, 'fetch', () => Promise.resolve(response));
+
+    const providerResponse = await provider.get<{ validJson: boolean }>('/endpoint/123', {
+      credentials: { apiKey: 'apikey#1111' },
+      logger: logger,
+      signal: new AbortController().signal,
+    });
+
+    assert.ok(providerResponse);
+    assert.equal(providerResponse.body, undefined);
+  });
+
+  it('returns streamable response on streaming get calls', async context => {
+    const response = new Response(`IMAGINE A HUGE PAYLOAD`, {
+      status: 200,
+      headers: { 'Content-Type': 'video/mp4' },
+    });
+
+    context.mock.method(global, 'fetch', () => Promise.resolve(response));
+
+    const providerResponse = await provider.streamingGet('/endpoint/123', {
+      credentials: { apiKey: 'apikey#1111' },
+      logger: logger,
+      signal: new AbortController().signal,
+    });
+
+    assert.ok(providerResponse);
+    assert.ok(providerResponse.body instanceof ReadableStream);
+  });
+
   it('throws on invalid json response', async context => {
     const response = new Response('{invalidJSON}', {
       status: 200,
+      headers: { 'Content-Type': 'application/json' },
     });
 
     context.mock.method(global, 'fetch', () => Promise.resolve(response));
@@ -285,10 +340,34 @@ describe('Provider', () => {
       error = e;
     }
 
-    assert.ok(error instanceof HttpErrors.BadRequestError);
+    assert.ok(error instanceof HttpErrors.HttpError);
     assert.equal(error.message, 'Invalid JSON response');
   });
 
+  it('throws on unexpected content-type response', async context => {
+    const response = new Response('text', {
+      status: 200,
+      headers: { 'Content-Type': 'application/text' },
+    });
+
+    context.mock.method(global, 'fetch', () => Promise.resolve(response));
+
+    let error;
+
+    try {
+      await provider.get('/endpoint/123', {
+        credentials: { apiKey: 'apikey#1111' },
+        logger: logger,
+        signal: new AbortController().signal,
+      });
+    } catch (e) {
+      error = e;
+    }
+
+    assert.ok(error instanceof HttpErrors.HttpError);
+    assert.equal(error.message, `Unsupported content-type. Expected 'application/json', got 'application/text'`);
+  });
+
   it('throws on status 400', async context => {
     const response = new Response('response body', {
       status: 400,