@tstdl/base 0.92.103 → 0.92.105

package/ai/ai.service.js

@@ -4,11 +4,14 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
     else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
     return c > 3 && r && Object.defineProperty(target, key, r), r;
 };
+var _a;
+var AiService_1;
 import { FinishReason, FunctionCallingMode as GoogleFunctionCallingMode, VertexAI } from '@google-cloud/vertexai';
 import { GoogleGenerativeAI } from '@google/generative-ai';
 import { NotSupportedError } from '../errors/not-supported.error.js';
 import { Singleton } from '../injector/decorators.js';
 import { inject, injectArgument } from '../injector/inject.js';
+import { Logger } from '../logger/logger.js';
 import { getShutdownSignal } from '../process-shutdown.js';
 import { DeferredPromise } from '../promise/deferred-promise.js';
 import { LazyPromise } from '../promise/lazy-promise.js';
@@ -38,9 +41,10 @@ const functionCallingModeMap = {
     force: GoogleFunctionCallingMode.ANY,
     none: GoogleFunctionCallingMode.NONE
 };
-let AiService = class AiService {
+let AiService = AiService_1 = class AiService {
     #options = injectArgument(this, { optional: true }) ?? inject(AiServiceOptions);
     #fileService = inject(AiFileService, this.#options);
+    #logger = inject(Logger, AiService_1.name);
     #genAI = (isDefined(this.#options.vertex)
         ? new VertexAI({ project: this.#options.vertex.project, location: this.#options.vertex.location, googleAuthOptions: { apiKey: this.#options.apiKey, keyFile: this.#options.keyFile } })
         : new GoogleGenerativeAI(assertDefinedPass(this.#options.apiKey, 'Api key not defined')));
@@ -196,6 +200,7 @@ Always output the content and tags in ${options?.targetLanguage ?? 'the same lan
         return mergeGenerationStreamItems(items, request.generationSchema);
     }
     async *generateStream(request) {
+        this.#logger.verbose('Generating...');
         const googleFunctionDeclarations = isDefined(request.functions) ? await this.convertFunctions(request.functions) : undefined;
         const generationConfig = {
             maxOutputTokens: request.generationOptions?.maxOutputTokens,
@@ -234,6 +239,7 @@ Always output the content and tags in ${options?.targetLanguage ?? 'the same lan
                 }
                 catch (error) {
                     if ((i < 20) && isError(error) && (error['status'] == 429)) {
+                        this.#logger.verbose('429 Too Many Requests - trying again in 15 seconds');
                         const canceled = await cancelableTimeout(15 * millisecondsPerSecond, getShutdownSignal());
                         if (!canceled) {
                             continue;
@@ -390,7 +396,7 @@ Always output the content and tags in ${options?.targetLanguage ?? 'the same lan
         return this.#genAI.getGenerativeModel({ model });
     }
 };
-AiService = __decorate([
+AiService = AiService_1 = __decorate([
     Singleton()
 ], AiService);
 export { AiService };

package/document-management/server/services/document-management.service.js

@@ -134,7 +134,7 @@ let DocumentManagementService = DocumentManagementService_1 = class DocumentMana
     processQueues(cancellationSignal) {
         this.extractionQueue.process({ concurrency: 1, cancellationSignal }, async (job) => {
             const [entry] = objectEntries(job.data);
-            this.logger.info(`Processing extraction job for ${entry?.[0]} "${entry?.[1]}"`);
+            this.logger.verbose(`Processing extraction job for ${entry?.[0]} ${entry?.[1]}`);
             await match(job.data)
                 .with({ documentId: P.string.select() }, async (documentId) => this.enrichDocument(documentId))
                 .with({ requestFileId: P.string.select() }, async (requestFileId) => this.enrichDocumentRequestFile(requestFileId))
@@ -142,7 +142,7 @@ let DocumentManagementService = DocumentManagementService_1 = class DocumentMana
                 .exhaustive();
         }, this.logger);
         this.assignmentQueue.process({ concurrency: 1, cancellationSignal }, async (job) => {
-            this.logger.info(`Processing assignment job "${job.data.requestAssignmentTaskId}"`);
+            this.logger.verbose(`Processing assignment job "${job.data.requestAssignmentTaskId}"`);
             await this.assignDocumentRequest(job.data.requestAssignmentTaskId);
         }, this.logger);
     }
@@ -554,6 +554,7 @@ let DocumentManagementService = DocumentManagementService_1 = class DocumentMana
     async enrichDocument(documentId) {
         const document = await this.documentService.load(documentId);
         const { properties, ...extractionResult } = await this.extractFileInformation(document.fileId, document.typeId);
+        this.logger.trace(`Applying extraction to document ${document.id}`);
         await this.documentService.transaction(async (documentService, transaction) => {
             await documentService.update(document.id, { ...extractionResult, validated: false });
             await this.setPropertyValues({ documentId, properties: properties }, transaction);
@@ -710,7 +711,10 @@ Ordne die Datei unter "file" der passenden Anforderungen unter "requests" zu. Gi
             const tmpFile = __addDisposableResource(env_1, await TemporaryFile.from(fileContentStream), true);
             const filePart = await this.#aiService.processFile({ path: tmpFile.path, mimeType: file.mimeType });
             const pages = file.mimeType.includes('pdf')
-                ? await tryIgnoreLogAsync(this.logger, async () => getPdfPageCount(tmpFile.path), null)
+                ? await tryIgnoreLogAsync(this.logger, async () => {
+                    this.logger.trace(`Extracting pdf page count for file ${fileId}`);
+                    return getPdfPageCount(tmpFile.path);
+                }, null)
                 : null;
             const types = await this.documentTypeService.session
                 .select({
@@ -724,6 +728,7 @@ Ordne die Datei unter "file" der passenden Anforderungen unter "requests" zu. Gi
                 .where(isString(assumeTypeId) ? eq(documentType.id, assumeTypeId) : undefined);
             const typeLabelEntries = types.map((type) => ({ id: type.typeId, label: `${type.categoryLabel} | ${type.typeGroup} | ${type.typeLabel}` }));
             const typeLabels = typeLabelEntries.map(({ label }) => label);
+            this.logger.trace(`Classifying document file ${fileId}`);
             const documentTypeGeneration = await this.#aiService.generate({
                 ...defaultGenerationOptions,
                 generationSchema: object({
@@ -763,6 +768,7 @@ Ordne die Datei unter "file" der passenden Anforderungen unter "requests" zu. Gi
                     ? {}
                     : { documentProperties: object(fromEntries(propertiesSchemaEntries)) })
             });
+            this.logger.trace(`Extracting document file ${fileId}`);
             const { json: extraction } = await this.#aiService.generate({
                 model: 'gemini-2.0-flash',
                 generationOptions: {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tstdl/base",
-  "version": "0.92.103",
+  "version": "0.92.105",
   "author": "Patrick Hein",
   "publishConfig": {
     "access": "public"

package/queue/queue.d.ts

@@ -78,12 +78,12 @@ export declare abstract class Queue<T> implements Resolvable<QueueArgument> {
     process({ concurrency, cancellationSignal }: {
         concurrency?: number;
         cancellationSignal: CancellationSignal;
-    }, handler: ProcessWorker<T>, errorHandler?: Logger | ((error: unknown) => void | Promise<void>)): void;
+    }, handler: ProcessWorker<T>, logger: Logger): void;
     processBatch({ batchSize, concurrency, cancellationSignal }: {
         batchSize?: number;
         concurrency?: number;
         cancellationSignal: CancellationSignal;
-    }, handler: ProcessBatchWorker<T>, errorHandler?: Logger | ((error: unknown) => void | Promise<void>)): void;
+    }, handler: ProcessBatchWorker<T>, logger: Logger): void;
     private processWorker;
     private processBatchWorker;
 }

package/queue/queue.js

@@ -1,4 +1,3 @@
-import { isDefined, isFunction } from '../utils/type-guards.js';
 import { millisecondsPerMinute } from '../utils/units.js';
 import { QueueEnqueueBatch } from './enqueue-batch.js';
 export const defaultJobPriority = 1000;
@@ -15,45 +14,39 @@ export class Queue {
     batch() {
         return new QueueEnqueueBatch(this);
     }
-    process({ concurrency = 1, cancellationSignal }, handler, errorHandler) {
-        const handleError = isFunction(errorHandler)
-            ? errorHandler
-            : isDefined(errorHandler)
-                ? (error) => errorHandler.error(error)
-                : undefined;
+    process({ concurrency = 1, cancellationSignal }, handler, logger) {
         for (let i = 0; i < concurrency; i++) {
-            void this.processWorker(cancellationSignal, handler, handleError);
+            void this.processWorker(cancellationSignal, handler, logger);
         }
     }
-    processBatch({ batchSize = 10, concurrency = 1, cancellationSignal }, handler, errorHandler) {
-        const handleError = isFunction(errorHandler)
-            ? errorHandler
-            : isDefined(errorHandler)
-                ? (error) => errorHandler.error(error)
-                : undefined;
+    processBatch({ batchSize = 10, concurrency = 1, cancellationSignal }, handler, logger) {
         for (let i = 0; i < concurrency; i++) {
-            void this.processBatchWorker(batchSize, cancellationSignal, handler, handleError);
+            void this.processBatchWorker(batchSize, cancellationSignal, handler, logger);
         }
     }
-    async processWorker(cancellationSignal, handler, errorHandler) {
+    async processWorker(cancellationSignal, handler, logger) {
         for await (const job of this.getConsumer(cancellationSignal)) {
+            logger?.verbose(`Processing job ${job.id}`);
             try {
                 await handler(job);
+                logger?.verbose(`Acknowledge job ${job.id}`);
                 await this.acknowledge(job);
             }
             catch (error) {
-                await errorHandler?.(error);
+                logger?.error(error);
             }
         }
     }
-    async processBatchWorker(size, cancellationSignal, handler, errorHandler) {
+    async processBatchWorker(size, cancellationSignal, handler, logger) {
         for await (const jobs of this.getBatchConsumer(size, cancellationSignal)) {
+            logger?.verbose(`Processing ${jobs.length} jobs`);
             try {
                 await handler(jobs);
+                logger?.verbose(`Acknowledge ${jobs.length} jobs`);
                 await this.acknowledgeMany(jobs);
             }
             catch (error) {
-                await errorHandler?.(error);
+                logger?.error(error);
             }
         }
     }