@tstdl/base 0.93.139 → 0.93.141

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (218) hide show
  1. package/README.md +166 -0
  2. package/ai/genkit/multi-region.plugin.js +5 -3
  3. package/ai/genkit/tests/multi-region.test.d.ts +1 -0
  4. package/ai/genkit/tests/multi-region.test.js +5 -2
  5. package/ai/parser/parser.js +2 -2
  6. package/ai/prompts/build.js +1 -0
  7. package/ai/prompts/instructions-formatter.d.ts +15 -2
  8. package/ai/prompts/instructions-formatter.js +36 -31
  9. package/ai/prompts/prompt-builder.js +5 -5
  10. package/ai/prompts/steering.d.ts +3 -2
  11. package/ai/prompts/steering.js +3 -1
  12. package/ai/tests/instructions-formatter.test.js +1 -0
  13. package/api/README.md +403 -0
  14. package/api/client/client.js +7 -13
  15. package/api/client/tests/api-client.test.js +10 -10
  16. package/api/default-error-handlers.js +1 -1
  17. package/api/response.d.ts +2 -2
  18. package/api/response.js +22 -33
  19. package/api/server/api-controller.d.ts +1 -1
  20. package/api/server/api-controller.js +3 -3
  21. package/api/server/api-request-token.provider.d.ts +1 -0
  22. package/api/server/api-request-token.provider.js +1 -0
  23. package/api/server/middlewares/allowed-methods.middleware.js +2 -1
  24. package/api/server/middlewares/content-type.middleware.js +2 -1
  25. package/api/types.d.ts +3 -2
  26. package/application/README.md +240 -0
  27. package/application/application.d.ts +1 -1
  28. package/application/application.js +3 -3
  29. package/application/providers.d.ts +20 -2
  30. package/application/providers.js +34 -7
  31. package/audit/README.md +267 -0
  32. package/audit/module.d.ts +5 -0
  33. package/audit/module.js +9 -1
  34. package/authentication/README.md +288 -0
  35. package/authentication/client/authentication.service.d.ts +12 -11
  36. package/authentication/client/authentication.service.js +21 -21
  37. package/authentication/client/http-client.middleware.js +2 -2
  38. package/authentication/server/module.d.ts +5 -0
  39. package/authentication/server/module.js +9 -1
  40. package/authentication/tests/authentication.api-controller.test.js +1 -1
  41. package/authentication/tests/authentication.api-request-token.provider.test.js +1 -1
  42. package/authentication/tests/authentication.client-error-handling.test.js +2 -1
  43. package/authentication/tests/authentication.client-service-refresh.test.js +5 -3
  44. package/authentication/tests/authentication.client-service.test.js +1 -1
  45. package/browser/README.md +401 -0
  46. package/cancellation/README.md +156 -0
  47. package/cancellation/tests/coverage.test.d.ts +1 -0
  48. package/cancellation/tests/coverage.test.js +49 -0
  49. package/cancellation/tests/leak.test.js +24 -29
  50. package/cancellation/tests/token.test.d.ts +1 -0
  51. package/cancellation/tests/token.test.js +136 -0
  52. package/cancellation/token.d.ts +53 -177
  53. package/cancellation/token.js +132 -208
  54. package/circuit-breaker/postgres/module.d.ts +1 -0
  55. package/circuit-breaker/postgres/module.js +5 -1
  56. package/context/README.md +174 -0
  57. package/cookie/README.md +161 -0
  58. package/css/README.md +157 -0
  59. package/data-structures/README.md +320 -0
  60. package/decorators/README.md +140 -0
  61. package/distributed-loop/README.md +231 -0
  62. package/distributed-loop/distributed-loop.js +1 -1
  63. package/document-management/README.md +403 -0
  64. package/document-management/server/configure.js +5 -1
  65. package/document-management/server/module.d.ts +1 -1
  66. package/document-management/server/module.js +1 -1
  67. package/document-management/server/services/document-management-ancillary.service.js +1 -1
  68. package/document-management/server/services/document-management.service.js +9 -7
  69. package/document-management/tests/ai-config-hierarchy.test.js +0 -5
  70. package/document-management/tests/document-management-ai-overrides.test.js +0 -1
  71. package/document-management/tests/document-management-core.test.js +2 -7
  72. package/document-management/tests/document-management.api.test.js +6 -7
  73. package/document-management/tests/document-statistics.service.test.js +11 -12
  74. package/document-management/tests/document-validation-ai-overrides.test.js +0 -1
  75. package/document-management/tests/document.service.test.js +3 -3
  76. package/document-management/tests/enum-helpers.test.js +2 -3
  77. package/dom/README.md +213 -0
  78. package/enumerable/README.md +259 -0
  79. package/enumeration/README.md +121 -0
  80. package/errors/README.md +267 -0
  81. package/examples/document-management/main.d.ts +1 -0
  82. package/examples/document-management/main.js +14 -11
  83. package/file/README.md +191 -0
  84. package/formats/README.md +210 -0
  85. package/function/README.md +144 -0
  86. package/http/README.md +318 -0
  87. package/http/client/adapters/undici.adapter.js +1 -1
  88. package/http/client/http-client-request.d.ts +6 -5
  89. package/http/client/http-client-request.js +8 -9
  90. package/http/server/node/node-http-server.js +1 -2
  91. package/image-service/README.md +137 -0
  92. package/injector/README.md +491 -0
  93. package/intl/README.md +113 -0
  94. package/json-path/README.md +182 -0
  95. package/jsx/README.md +154 -0
  96. package/key-value-store/README.md +191 -0
  97. package/key-value-store/postgres/module.d.ts +1 -0
  98. package/key-value-store/postgres/module.js +5 -1
  99. package/lock/README.md +249 -0
  100. package/lock/postgres/module.d.ts +1 -0
  101. package/lock/postgres/module.js +5 -1
  102. package/lock/web/web-lock.js +119 -47
  103. package/logger/README.md +287 -0
  104. package/mail/README.md +256 -0
  105. package/mail/module.d.ts +5 -1
  106. package/mail/module.js +11 -6
  107. package/memory/README.md +144 -0
  108. package/message-bus/README.md +244 -0
  109. package/message-bus/message-bus-base.js +1 -1
  110. package/module/README.md +182 -0
  111. package/module/module.d.ts +1 -1
  112. package/module/module.js +77 -17
  113. package/module/modules/web-server.module.js +3 -4
  114. package/notification/server/module.d.ts +1 -0
  115. package/notification/server/module.js +5 -1
  116. package/notification/tests/notification-flow.test.js +2 -2
  117. package/notification/tests/notification-type.service.test.js +24 -15
  118. package/object-storage/README.md +300 -0
  119. package/openid-connect/README.md +274 -0
  120. package/orm/README.md +423 -0
  121. package/orm/decorators.d.ts +5 -1
  122. package/orm/decorators.js +1 -1
  123. package/orm/server/drizzle/schema-converter.js +17 -30
  124. package/orm/server/encryption.d.ts +0 -1
  125. package/orm/server/encryption.js +1 -4
  126. package/orm/server/index.d.ts +1 -6
  127. package/orm/server/index.js +1 -6
  128. package/orm/server/migration.d.ts +19 -0
  129. package/orm/server/migration.js +72 -0
  130. package/orm/server/repository.d.ts +1 -1
  131. package/orm/server/transaction.d.ts +5 -10
  132. package/orm/server/transaction.js +22 -26
  133. package/orm/server/transactional.js +3 -3
  134. package/orm/tests/database-migration.test.d.ts +1 -0
  135. package/orm/tests/database-migration.test.js +82 -0
  136. package/orm/tests/encryption.test.js +3 -4
  137. package/orm/utils.d.ts +17 -2
  138. package/orm/utils.js +49 -1
  139. package/package.json +9 -6
  140. package/password/README.md +164 -0
  141. package/pdf/README.md +246 -0
  142. package/polyfills.js +1 -0
  143. package/pool/README.md +198 -0
  144. package/process/README.md +237 -0
  145. package/promise/README.md +252 -0
  146. package/promise/cancelable-promise.js +1 -1
  147. package/random/README.md +193 -0
  148. package/rate-limit/postgres/module.d.ts +1 -0
  149. package/rate-limit/postgres/module.js +5 -1
  150. package/reflection/README.md +305 -0
  151. package/reflection/decorator-data.js +11 -12
  152. package/rpc/README.md +386 -0
  153. package/rxjs-utils/README.md +262 -0
  154. package/schema/README.md +342 -0
  155. package/serializer/README.md +342 -0
  156. package/signals/implementation/README.md +134 -0
  157. package/sse/README.md +278 -0
  158. package/task-queue/README.md +293 -0
  159. package/task-queue/postgres/drizzle/{0000_simple_invisible_woman.sql → 0000_wakeful_sunspot.sql} +22 -14
  160. package/task-queue/postgres/drizzle/meta/0000_snapshot.json +160 -82
  161. package/task-queue/postgres/drizzle/meta/_journal.json +2 -2
  162. package/task-queue/postgres/module.d.ts +1 -0
  163. package/task-queue/postgres/module.js +5 -1
  164. package/task-queue/postgres/schemas.d.ts +9 -6
  165. package/task-queue/postgres/schemas.js +4 -3
  166. package/task-queue/postgres/task-queue.d.ts +4 -13
  167. package/task-queue/postgres/task-queue.js +462 -355
  168. package/task-queue/postgres/task.model.d.ts +12 -5
  169. package/task-queue/postgres/task.model.js +51 -25
  170. package/task-queue/task-context.d.ts +2 -2
  171. package/task-queue/task-context.js +8 -8
  172. package/task-queue/task-queue.d.ts +53 -19
  173. package/task-queue/task-queue.js +121 -55
  174. package/task-queue/tests/cascading-cancellations.test.d.ts +1 -0
  175. package/task-queue/tests/cascading-cancellations.test.js +38 -0
  176. package/task-queue/tests/complex.test.js +45 -229
  177. package/task-queue/tests/coverage-branch.test.d.ts +1 -0
  178. package/task-queue/tests/coverage-branch.test.js +407 -0
  179. package/task-queue/tests/coverage-enhancement.test.d.ts +1 -0
  180. package/task-queue/tests/coverage-enhancement.test.js +144 -0
  181. package/task-queue/tests/dag-dependencies.test.d.ts +1 -0
  182. package/task-queue/tests/dag-dependencies.test.js +41 -0
  183. package/task-queue/tests/dependencies.test.js +28 -26
  184. package/task-queue/tests/extensive-dependencies.test.js +64 -139
  185. package/task-queue/tests/fan-out-spawning.test.d.ts +1 -0
  186. package/task-queue/tests/fan-out-spawning.test.js +53 -0
  187. package/task-queue/tests/idempotent-replacement.test.d.ts +1 -0
  188. package/task-queue/tests/idempotent-replacement.test.js +61 -0
  189. package/task-queue/tests/missing-idempotent-tasks.test.d.ts +1 -0
  190. package/task-queue/tests/missing-idempotent-tasks.test.js +38 -0
  191. package/task-queue/tests/queue.test.js +128 -8
  192. package/task-queue/tests/worker.test.js +39 -16
  193. package/task-queue/tests/zombie-parent.test.d.ts +1 -0
  194. package/task-queue/tests/zombie-parent.test.js +45 -0
  195. package/task-queue/tests/zombie-recovery.test.d.ts +1 -0
  196. package/task-queue/tests/zombie-recovery.test.js +51 -0
  197. package/templates/README.md +287 -0
  198. package/test5.js +5 -5
  199. package/testing/README.md +157 -0
  200. package/testing/integration-setup.d.ts +4 -4
  201. package/testing/integration-setup.js +54 -29
  202. package/text/README.md +346 -0
  203. package/text/localization.service.js +2 -2
  204. package/threading/README.md +238 -0
  205. package/types/README.md +311 -0
  206. package/utils/README.md +322 -0
  207. package/utils/async-iterable-helpers/observable-iterable.d.ts +1 -1
  208. package/utils/async-iterable-helpers/observable-iterable.js +4 -8
  209. package/utils/async-iterable-helpers/take-until.js +4 -4
  210. package/utils/backoff.js +89 -30
  211. package/utils/file-reader.js +1 -2
  212. package/utils/retry-with-backoff.js +1 -1
  213. package/utils/timer.d.ts +1 -1
  214. package/utils/timer.js +5 -7
  215. package/utils/timing.d.ts +1 -1
  216. package/utils/timing.js +2 -4
  217. package/utils/z-base32.d.ts +1 -0
  218. package/utils/z-base32.js +1 -0
@@ -0,0 +1,403 @@
1
+ # Document Management Module
2
+
3
+ The `@tstdl/base/document-management` module provides a comprehensive, AI-enhanced system for managing document lifecycles. It features hierarchical collections, automated classification and data extraction, structured validation workflows, and a robust permission system.
4
+
5
+ ## Table of Contents
6
+
7
+ - [✨ Features](#-features)
8
+ - [Core Concepts](#core-concepts)
9
+ - [🚀 Basic Usage](#-basic-usage)
10
+ - [Server Configuration](#server-configuration)
11
+ - [Client API Interaction](#client-api-interaction)
12
+ - [🔧 Advanced Topics](#-advanced-topics)
13
+ - [Defining Taxonomy (Categories & Types)](#defining-taxonomy-categories--types)
14
+ - [Implementing Authorization](#implementing-authorization)
15
+ - [Context Metadata (Ancillary Service)](#context-metadata-ancillary-service)
16
+ - [Advanced AI Configuration](#advanced-ai-configuration)
17
+ - [Document Requests & Templates](#document-requests--templates)
18
+ - [Retrieving Statistics](#retrieving-statistics)
19
+ - [Real-time Updates](#real-time-updates)
20
+ - [📚 API](#-api)
21
+
22
+ ## ✨ Features
23
+
24
+ - **Hierarchical Collections**: Organize documents into logical groups (e.g., by tenant, case, or project) with support for infinitely nested sub-collections.
25
+ - **AI-Powered Workflow**: Asynchronous processing pipeline including:
26
+ - **Classification**: Automatically determines document type.
27
+ - **Extraction**: Extracts structured properties (dates, amounts, etc.) and tags.
28
+ - **Assignment**: Suggests target collections or requests based on content.
29
+ - **Human-in-the-Loop**: Multi-stage verification process. AI results are presented in "Review" states for manual verification and correction before proceeding.
30
+ - **Structured Taxonomy**: Define multi-level Categories, Document Types, and strongly-typed Properties (Text, Number, Date, Boolean).
31
+ - **Document Requests**: Formalize requirements for specific documents, track fulfillment, and use templates for recurring needs (e.g., onboarding packages).
32
+ - **Validation Framework**: Extensible validation logic integrated into the workflow. Support for automated checks and multi-document consistency.
33
+ - **Real-time Synchronization**: Built-in SSE support for live updates of document states, requests, and collection data.
34
+ - **Secure File Handling**: Direct-to-object-storage uploads via signed URLs and on-the-fly preview generation for various file types.
35
+ - **Comprehensive Statistics**: Built-in reporting on document counts, approval states, type distribution, and validation failures.
36
+
37
+ ## Core Concepts
38
+
39
+ ### Collections & Assignments
40
+
41
+ A **Collection** is the primary container for documents. Documents are not "stored" inside a collection physically but are **assigned** to them. A single document can be assigned to multiple collections.
42
+
43
+ ### Taxonomy
44
+
45
+ The system relies on a structured definition of data:
46
+
47
+ 1. **Categories**: High-level grouping (e.g., "Financial", "Legal"). Supports hierarchical nesting (sub-categories).
48
+ 2. **Types**: Specific document kinds (e.g., "Invoice", "Contract") belonging to a category.
49
+ 3. **Properties**: Typed data fields (e.g., "Invoice Amount", "Due Date") attached to specific Document Types.
50
+
51
+ ### The Workflow & Immutability
52
+
53
+ Every uploaded document goes through a state machine managed by the `DocumentWorkflowService`:
54
+
55
+ 1. **Classification**: AI determines the `DocumentType`.
56
+ 2. **Extraction**: AI populates `DocumentProperties` and tags.
57
+ 3. **Assignment**: AI suggests collections or requests if not explicitly provided during upload.
58
+ 4. **Validation**: Runs automated checks (e.g., "Is this a valid ID card?").
59
+
60
+ Between steps, the document may enter a **Review** state (e.g., `Waiting for Review (Analysis)`), requiring a user to confirm or correct the data via the API (`proceedDocumentWorkflow`).
61
+
62
+ **Immutability**: Once a document is moved to the **Approved** state, it becomes read-only. This ensures data integrity for processed documents. Any errors discovered after approval require the document to be archived and a new version uploaded.
63
+
64
+ ### Assignments & Archiving
65
+
66
+ Documents are linked to collections via **Assignments**. A document can be assigned to many collections simultaneously. If a document is no longer needed in a specific context, you can **archive** the assignment. This hides the document from that collection's views without deleting the document entity or affecting its assignments in other collections.
67
+
68
+ ## 🚀 Basic Usage
69
+
70
+ ### Server Configuration
71
+
72
+ To use the module, you must configure it during your application bootstrap. This involves registering the module and providing implementations for authorization and metadata resolution.
73
+
74
+ ```typescript
75
+ import { configureDocumentManagement } from '@tstdl/base/document-management/server';
76
+ import { AppAncillaryService } from './services/app-ancillary.service.js';
77
+ import { AppAuthorizationService } from './services/app-authorization.service.js';
78
+
79
+ // In your application bootstrap function
80
+ export async function bootstrap() {
81
+ configureDocumentManagement({
82
+ // Service to resolve collection names/groups from your app's data
83
+ ancillaryService: AppAncillaryService,
84
+
85
+ // Service to handle permissions
86
+ authorizationService: AppAuthorizationService,
87
+
88
+ // Object Storage module names (must be configured via configureObjectStorage)
89
+ fileObjectStorageModule: 'documents',
90
+ fileUploadObjectStorageModule: 'document-uploads',
91
+ filePreviewObjectStorageModule: 'document-previews',
92
+
93
+ // Optional database config override
94
+ // database: { ... }
95
+ });
96
+ }
97
+ ```
98
+
99
+ ### Client API Interaction
100
+
101
+ The `DocumentManagementApi` provides a type-safe client for all operations.
102
+
103
+ #### 1. Uploading a Document
104
+
105
+ Uploading is a two-step process: get a signed URL, upload the binary, then create the entity.
106
+
107
+ ```typescript
108
+ import { inject } from '@tstdl/base/injector';
109
+ import { DocumentManagementApi } from '@tstdl/base/document-management';
110
+
111
+ async function uploadInvoice(file: File, collectionId: string) {
112
+ const api = inject(DocumentManagementApi);
113
+
114
+ // 1. Initiate upload to get a secure URL
115
+ const { uploadId, uploadUrl } = await api.initiateDocumentUpload({
116
+ contentLength: file.size,
117
+ });
118
+
119
+ // 2. Upload the file content directly to storage
120
+ await fetch(uploadUrl, { method: 'PUT', body: file });
121
+
122
+ // 3. Create the document entity
123
+ const document = await api.createDocument({
124
+ uploadId,
125
+ originalFileName: file.name,
126
+ assignment: {
127
+ collections: [collectionId], // Assign to a specific collection
128
+ },
129
+ // Optional: Provide initial metadata if known
130
+ // title: 'My Invoice',
131
+ // typeId: 'invoice-type-id'
132
+ });
133
+
134
+ console.log('Document created:', document.id);
135
+ }
136
+ ```
137
+
138
+ #### 2. Loading Data
139
+
140
+ Fetch all documents, requests, and metadata for a set of collections.
141
+
142
+ ```typescript
143
+ import { inject } from '@tstdl/base/injector';
144
+ import { DocumentManagementApi, toEnrichedDocumentManagementData } from '@tstdl/base/document-management';
145
+
146
+ async function loadDashboard(collectionId: string) {
147
+ const api = inject(DocumentManagementApi);
148
+
149
+ // Load raw data
150
+ const rawData = await api.loadData({ collectionIds: [collectionId] });
151
+
152
+ // Convert to enriched view model for easier consumption (resolves references)
153
+ const data = toEnrichedDocumentManagementData(rawData);
154
+
155
+ for (const doc of data.documents) {
156
+ console.log(`Document: ${doc.title} (${doc.type?.label})`);
157
+ console.log(`Status: ${doc.approval}`);
158
+
159
+ // Access typed properties easily
160
+ const amount = doc.properties.find((p) => p.label === 'Amount')?.value;
161
+ }
162
+ }
163
+ ```
164
+
165
+ ## 🔧 Advanced Topics
166
+
167
+ ### Defining Taxonomy (Categories & Types)
168
+
169
+ You should seed your database with a defined taxonomy. This is typically done in a startup script or migration utility using `DocumentManagementService`.
170
+
171
+ ```typescript
172
+ import { DocumentManagementService } from '@tstdl/base/document-management/server';
173
+ import { DocumentPropertyDataType } from '@tstdl/base/document-management';
174
+ import { defineEnum } from '@tstdl/base/enumeration';
175
+ import { inject } from '@tstdl/base/injector';
176
+
177
+ // Define Enums for type safety
178
+ const MyCategory = defineEnum('MyCategory', { Finance: 'finance' });
179
+ const MyType = defineEnum('MyType', { Invoice: 'invoice' });
180
+ const MyProp = defineEnum('MyProp', { Amount: 'amount', DueDate: 'due-date' });
181
+
182
+ async function seedTaxonomy(tenantId: string) {
183
+ const service = inject(DocumentManagementService);
184
+
185
+ await service.initializeCategoriesAndTypes(tenantId, {
186
+ categoryLabels: { [MyCategory.Finance]: 'Financial Documents' },
187
+ categoryParents: { [MyCategory.Finance]: null },
188
+
189
+ typeLabels: { [MyType.Invoice]: 'Incoming Invoice' },
190
+ typeCategories: { [MyType.Invoice]: MyCategory.Finance },
191
+
192
+ propertyConfigurations: {
193
+ [MyProp.Amount]: [DocumentPropertyDataType.Decimal, 'Total Amount'],
194
+ [MyProp.DueDate]: [DocumentPropertyDataType.Date, 'Payment Due Date'],
195
+ },
196
+
197
+ typeProperties: {
198
+ [MyType.Invoice]: [MyProp.Amount, MyProp.DueDate],
199
+ },
200
+ });
201
+ }
202
+ ```
203
+
204
+ ### Implementing Authorization
205
+
206
+ You must implement `DocumentManagementAuthorizationService` to control access. This service is called by the API layer.
207
+
208
+ ```typescript
209
+ import { DocumentManagementAuthorizationService } from '@tstdl/base/document-management';
210
+ import { Singleton } from '@tstdl/base/injector';
211
+
212
+ // Assuming your app uses a specific token type
213
+ type MyToken = { payload: { userId: string; tenantId: string; roles: string[] } };
214
+
215
+ @Singleton()
216
+ export class AppAuthorizationService extends DocumentManagementAuthorizationService<MyToken> {
217
+ getTenantId(token?: MyToken): string {
218
+ return token?.payload.tenantId ?? '';
219
+ }
220
+
221
+ getSubject(token?: MyToken): string {
222
+ return token?.payload.userId ?? '';
223
+ }
224
+
225
+ async canReadCollection(collectionId: string, token?: MyToken): Promise<boolean> {
226
+ // Implement your logic: check if user has access to this collection
227
+ return true;
228
+ }
229
+
230
+ async canApproveDocument(documentId: string, token?: MyToken): Promise<boolean> {
231
+ return token?.payload.roles.includes('manager') ?? false;
232
+ }
233
+
234
+ // ... implement other abstract methods (canCreateDocuments, canAssignDocuments, etc.)
235
+ }
236
+ ```
237
+
238
+ ### Context Metadata (Ancillary Service)
239
+
240
+ The `DocumentManagementAncillaryService` allows you to inject application-specific data (like names) into the document management views, as the module itself only knows about Collection IDs.
241
+
242
+ ```typescript
243
+ import { DocumentManagementAncillaryService } from '@tstdl/base/document-management/server';
244
+ import { DocumentCollection, DocumentCollectionMetadata } from '@tstdl/base/document-management';
245
+ import { Singleton, inject } from '@tstdl/base/injector';
246
+ import { UserRepository } from './user.repository.js';
247
+
248
+ @Singleton()
249
+ export class AppAncillaryService extends DocumentManagementAncillaryService {
250
+ readonly #userRepo = inject(UserRepository);
251
+
252
+ async resolveMetadata(collections: DocumentCollection[]): Promise<DocumentCollectionMetadata[]> {
253
+ // Example: If collection.id corresponds to a User ID
254
+ const userIds = collections.map((c) => c.id);
255
+ const users = await this.#userRepo.loadMany(userIds);
256
+
257
+ return collections.map((collection) => {
258
+ const user = users.find((u) => u.id == collection.id);
259
+ return {
260
+ name: user ? `${user.firstName} ${user.lastName}` : 'Unknown Collection',
261
+ group: 'User Files',
262
+ };
263
+ });
264
+ }
265
+ }
266
+ ```
267
+
268
+ ### Advanced AI Configuration
269
+
270
+ The module allows for fine-grained control over AI behavior during classification and data extraction. Configuration follows a strict hierarchy:
271
+
272
+ 1. **Document Type**: Config linked to a specific `DocumentType` key.
273
+ 2. **Document Category**: Config linked to a specific `DocumentCategory` key.
274
+ 3. **Step-Specific Global**: Global config for a `DocumentWorkflowStep`.
275
+ 4. **Global Defaults**: Fallback module configuration.
276
+
277
+ #### Structured Overrides
278
+
279
+ For each core field (`title`, `subtitle`, `date`, `summary`, `tags`) and dynamic properties, you can provide granular instructions using the `InstructionOverride` type:
280
+
281
+ - **Simple Way**: Provide a `format` string (e.g., `INV-{{Date}}`).
282
+ - **Advanced Way**: Provide custom `content` and a `strategy` (`append` or `replace`).
283
+
284
+ ```typescript
285
+ // Example configuration in your AI Provider
286
+ override getGlobalConfiguration() {
287
+ return {
288
+ documentTypes: {
289
+ 'invoice': {
290
+ extraction: {
291
+ title: { format: 'Invoice - {{Vendor}} - {{Date}}' },
292
+ summary: { content: 'Focus on individual line items.', strategy: 'append' }
293
+ }
294
+ }
295
+ },
296
+ defaults: {
297
+ classification: { content: 'Always prefer finance categories if an IBAN is found.', strategy: 'append' }
298
+ }
299
+ };
300
+ }
301
+ ```
302
+
303
+ ### Document Requests & Templates
304
+
305
+ Create requests to ask users for specific documents.
306
+
307
+ ```typescript
308
+ import { inject } from '@tstdl/base/injector';
309
+ import { DocumentManagementApi } from '@tstdl/base/document-management';
310
+
311
+ async function requestDocument() {
312
+ const api = inject(DocumentManagementApi);
313
+
314
+ // Create a single request
315
+ await api.createDocumentRequest({
316
+ collectionIds: ['target-collection-id'],
317
+ typeId: 'invoice-type-id',
318
+ comment: 'Please upload the invoice for March.',
319
+ });
320
+
321
+ // Or apply a pre-defined template
322
+ await api.applyDocumentRequestsTemplate({
323
+ id: 'onboarding-template-id',
324
+ collectionIds: ['new-employee-collection-id'],
325
+ });
326
+ }
327
+ ```
328
+
329
+ ### Retrieving Statistics
330
+
331
+ The module provides a flexible API to retrieve statistical insights about documents, such as counts, storage usage, and data quality metrics.
332
+
333
+ ```typescript
334
+ import { inject } from '@tstdl/base/injector';
335
+ import { DocumentManagementApi } from '@tstdl/base/document-management';
336
+
337
+ async function getStats(collectionId: string) {
338
+ const api = inject(DocumentManagementApi);
339
+
340
+ const stats = await api.getStatistics({
341
+ includeTotalCount: true,
342
+ includeApprovalBreakdown: true,
343
+ includeStorageUsage: true,
344
+ includeDataQuality: true,
345
+ collectionIds: [collectionId], // Optional: Filter by specific collections
346
+ });
347
+
348
+ console.log(`Total Documents: ${stats.totalCount}`);
349
+ console.log(`Storage Used: ${stats.storageUsage} bytes`);
350
+ console.log(`Approved: ${stats.approvalBreakdown?.approved}`);
351
+ console.log(`Documents missing type: ${stats.dataQuality?.missingType}`);
352
+ }
353
+ ```
354
+
355
+ ### Real-time Updates
356
+
357
+ The module supports Server-Sent Events (SSE) to push updates to the client when documents or collections change.
358
+
359
+ ```typescript
360
+ import { inject } from '@tstdl/base/injector';
361
+ import { DocumentManagementApi } from '@tstdl/base/document-management';
362
+
363
+ async function watchCollection(collectionId: string) {
364
+ const api = inject(DocumentManagementApi);
365
+
366
+ // Returns an AsyncIterable that yields new data whenever changes occur
367
+ const stream = api.loadDataStream({ collectionIds: [collectionId] });
368
+
369
+ for await (const data of stream) {
370
+ console.log('Data updated:', data);
371
+ // Update UI...
372
+ }
373
+ }
374
+ ```
375
+
376
+ ## 📚 API
377
+
378
+ ### Client / Shared
379
+
380
+ | Export | Description |
381
+ | :--------------------------------------- | :------------------------------------------------------------------------------ |
382
+ | `DocumentManagementApi` | Main client class for API interaction. |
383
+ | `DocumentManagementData` | DTO containing all loaded data (collections, documents, types, etc.). |
384
+ | `toEnrichedDocumentManagementData` | Helper to convert raw DTO into an object graph with resolved references. |
385
+ | `DocumentView` | Represents a document with its properties, tags, and workflow status. |
386
+ | `DocumentRequestView` | Represents a request for a document. |
387
+ | `DocumentPropertyDataType` | Enum defining supported property types (Text, Integer, Decimal, Boolean, Date). |
388
+ | `DocumentManagementAuthorizationService` | Abstract base class for authorization logic. |
389
+
390
+ ### Server
391
+
392
+ | Export | Description |
393
+ | :----------------------------------- | :------------------------------------------------------------- |
394
+ | `configureDocumentManagement` | Function to configure the module in the application bootstrap. |
395
+ | `DocumentManagementService` | Core service for managing data and initializing taxonomy. |
396
+ | `DocumentService` | Service for creating and updating documents. |
397
+ | `DocumentFileService` | Service for handling file uploads, storage, and previews. |
398
+ | `DocumentRequestService` | Service for managing document requests and templates. |
399
+ | `DocumentStatisticsService` | Service for retrieving document statistics. |
400
+ | `DocumentWorkflowService` | Service for managing the document processing workflow. |
401
+ | `DocumentManagementAncillaryService` | Abstract base class for resolving collection metadata. |
402
+ | `DocumentValidationExecutor` | Abstract base class for implementing custom validation logic. |
403
+ | `AiValidationExecutor` | Base class for AI-driven validation executors. |
@@ -1,7 +1,8 @@
1
1
  import { Injector } from '../../injector/injector.js';
2
+ import { registerDatabaseMigration } from '../../orm/server/index.js';
2
3
  import { isDefined } from '../../utils/type-guards.js';
3
4
  import { DocumentManagementAuthorizationService } from '../authorization/document-management-authorization.service.js';
4
- import { DocumentManagementConfiguration } from './module.js';
5
+ import { DocumentManagementConfiguration, migrateDocumentManagementSchema } from './module.js';
5
6
  import { DocumentManagementAiProviderService, DocumentManagementAncillaryService } from './services/index.js';
6
7
  export function configureDocumentManagement(configuration) {
7
8
  const targetInjector = configuration.injector ?? Injector;
@@ -11,4 +12,7 @@ export function configureDocumentManagement(configuration) {
11
12
  if (isDefined(configuration.aiProvider)) {
12
13
  targetInjector.register(DocumentManagementAiProviderService, { useToken: configuration.aiProvider });
13
14
  }
15
+ if (configuration.autoMigrate != false) {
16
+ registerDatabaseMigration('DocumentManagement', migrateDocumentManagementSchema, { injector: configuration.injector });
17
+ }
14
18
  }
@@ -1,4 +1,3 @@
1
- import './schemas.js';
2
1
  import { type InjectionToken } from '../../injector/index.js';
3
2
  import { type DatabaseConfig } from '../../orm/server/index.js';
4
3
  import type { DocumentManagementAuthorizationService } from '../authorization/document-management-authorization.service.js';
@@ -14,5 +13,6 @@ export declare class DocumentManagementConfiguration {
14
13
  database?: DatabaseConfig;
15
14
  maxFileSize?: number;
16
15
  skipAi?: boolean;
16
+ autoMigrate?: boolean;
17
17
  }
18
18
  export declare function migrateDocumentManagementSchema(): Promise<void>;
@@ -1,4 +1,3 @@
1
- import './schemas.js';
2
1
  import { inject } from '../../injector/index.js';
3
2
  import { Database, migrate } from '../../orm/server/index.js';
4
3
  export class DocumentManagementConfiguration {
@@ -11,6 +10,7 @@ export class DocumentManagementConfiguration {
11
10
  database;
12
11
  maxFileSize;
13
12
  skipAi;
13
+ autoMigrate;
14
14
  }
15
15
  ;
16
16
  export async function migrateDocumentManagementSchema() {
@@ -6,6 +6,6 @@ export class DocumentManagementAncillaryService {
6
6
  * @returns The resolved file name for the document, without the file extension.
7
7
  */
8
8
  getDocumentFilename(document) {
9
- return document.title ?? document.originalFileName ?? document.id;
9
+ return document.title ?? document.originalFileName?.slice(0, document.originalFileName.lastIndexOf('.')) ?? document.id;
10
10
  }
11
11
  }
@@ -8,18 +8,17 @@ var _a;
8
8
  var DocumentManagementService_1;
9
9
  import { and, eq } from 'drizzle-orm';
10
10
  import { union } from 'drizzle-orm/pg-core';
11
+ import { filter, firstValueFrom, race, share, takeUntil } from 'rxjs';
11
12
  import { Enumerable } from '../../../enumerable/index.js';
12
13
  import { BadRequestError } from '../../../errors/bad-request.error.js';
13
14
  import { inject } from '../../../injector/index.js';
14
15
  import { Logger } from '../../../logger/logger.js';
15
16
  import { Transactional, injectRepository, injectTransactional } from '../../../orm/server/index.js';
16
- import { DeferredPromise } from '../../../promise/deferred-promise.js';
17
17
  import { distinct } from '../../../utils/array/index.js';
18
18
  import { compareByValueSelection } from '../../../utils/comparison.js';
19
19
  import { groupToMap, groupToSingleMap } from '../../../utils/iterable-helpers/index.js';
20
20
  import { fromEntries, objectEntries } from '../../../utils/object/index.js';
21
21
  import { assertDefinedPass, isDefined, isNotNull, isNull, isUndefined } from '../../../utils/type-guards.js';
22
- import { filter, merge } from 'rxjs';
23
22
  import { DocumentAssignmentScope, DocumentAssignmentTask, DocumentCategory, DocumentCollectionAssignment, DocumentRequest, DocumentRequestCollectionAssignment, DocumentRequestTemplate, DocumentRequestsTemplate, DocumentType, DocumentTypeProperty, DocumentValidationExecution, DocumentWorkflowStep } from '../../models/index.js';
24
23
  import { documentAssignmentScope, documentAssignmentTask, documentCollectionAssignment, documentRequest, documentRequestCollectionAssignment } from '../schemas.js';
25
24
  import { DocumentCategoryTypeService } from './document-category-type.service.js';
@@ -72,14 +71,17 @@ let DocumentManagementService = DocumentManagementService_1 = class DocumentMana
72
71
  return result.map((row) => row.collectionId);
73
72
  }
74
73
  async *loadDataStream(tenantId, collectionIds, cancellationSignal) {
75
- const continuePromise = new DeferredPromise();
76
- const newData$ = this.#observationService.collectionsChangedMessageBus.allMessages$.pipe(filter((changedCollectionIds) => collectionIds.some((id) => changedCollectionIds.includes(id))));
77
- const subscription = merge(newData$, cancellationSignal).subscribe(() => continuePromise.resolveIfPending());
74
+ const collectionIdsSet = new Set(collectionIds);
75
+ let hasNewData = false;
76
+ const newData$ = this.#observationService.collectionsChangedMessageBus.allMessages$.pipe(filter((changedCollectionIds) => changedCollectionIds.some((id) => collectionIdsSet.has(id))), takeUntil(cancellationSignal), share());
77
+ const subscription = newData$.subscribe(() => hasNewData = true);
78
78
  try {
79
79
  while (cancellationSignal.isUnset) {
80
+ hasNewData = false;
80
81
  yield await this.loadData(tenantId, collectionIds);
81
- await continuePromise;
82
- continuePromise.reset();
82
+ if (!hasNewData && cancellationSignal.isUnset) {
83
+ await firstValueFrom(race(cancellationSignal, newData$));
84
+ }
83
85
  }
84
86
  }
85
87
  finally {
@@ -8,7 +8,6 @@ import { TestDocumentManagementAncillaryService } from './helper.js';
8
8
  describe('DocumentManagementAiService Hierarchy', () => {
9
9
  let injector;
10
10
  let aiService;
11
- let flashModel;
12
11
  const tenantId = crypto.randomUUID();
13
12
  const mockAiProvider = {
14
13
  getGlobalConfiguration: vi.fn(),
@@ -20,12 +19,8 @@ describe('DocumentManagementAiService Hierarchy', () => {
20
19
  };
21
20
  beforeAll(async () => {
22
21
  ({ injector } = await setupIntegrationTest({
23
- orm: { schema: 'document_management' },
24
22
  modules: { messageBus: true, signals: true, objectStorage: true, documentManagement: true },
25
23
  }));
26
- runInInjectionContext(injector, () => {
27
- flashModel = injectModel('gemini-2.5-flash');
28
- });
29
24
  injector.register(DocumentManagementAiProviderService, { useValue: mockAiProvider });
30
25
  // Ancillary and configuration are already registered by setupIntegrationTest, but we can override if needed
31
26
  injector.register(DocumentManagementAncillaryService, { useToken: TestDocumentManagementAncillaryService });
@@ -22,7 +22,6 @@ describe('DocumentManagementAiService Overrides', () => {
22
22
  };
23
23
  beforeAll(async () => {
24
24
  ({ injector } = await setupIntegrationTest({
25
- orm: { schema: 'document_management' },
26
25
  modules: { messageBus: true, signals: true, objectStorage: true },
27
26
  }));
28
27
  runInInjectionContext(injector, () => {
@@ -5,7 +5,6 @@ import { ObjectStorage } from '../../object-storage/index.js';
5
5
  import { TaskQueue } from '../../task-queue/index.js';
6
6
  import { clearTenantData, setupIntegrationTest } from '../../testing/index.js';
7
7
  import { configureDocumentManagement } from '../server/configure.js';
8
- import { migrateDocumentManagementSchema } from '../server/module.js';
9
8
  import { DocumentCategoryTypeService } from '../server/services/document-category-type.service.js';
10
9
  import { DocumentCollectionService } from '../server/services/document-collection.service.js';
11
10
  import { DocumentManagementAiService } from '../server/services/document-management-ai.service.js';
@@ -23,13 +22,11 @@ describe('Document Management Core', () => {
23
22
  let requestService;
24
23
  let managementService;
25
24
  let categoryTypeService;
26
- let workflowService;
27
- let propertyService;
28
25
  const schema = 'document_management';
29
26
  const tenantId = crypto.randomUUID();
30
27
  beforeAll(async () => {
31
28
  ({ injector, database } = await setupIntegrationTest({
32
- modules: { taskQueue: false, messageBus: true }, // Disabled TaskQueue to avoid background noise
29
+ modules: { taskQueue: false, messageBus: true, documentManagement: true }, // Disabled TaskQueue to avoid background noise
33
30
  orm: { schema },
34
31
  }));
35
32
  injector.register(GenkitModuleOptions, { useValue: {} });
@@ -58,15 +55,13 @@ describe('Document Management Core', () => {
58
55
  fileObjectStorageModule: 'documents',
59
56
  fileUploadObjectStorageModule: 'document-uploads',
60
57
  filePreviewObjectStorageModule: 'document-previews',
58
+ injector,
61
59
  });
62
- await runInInjectionContext(injector, migrateDocumentManagementSchema);
63
60
  documentService = await injector.resolveAsync(DocumentService);
64
61
  collectionService = await injector.resolveAsync(DocumentCollectionService);
65
62
  requestService = await injector.resolveAsync(DocumentRequestService);
66
63
  managementService = await injector.resolveAsync(DocumentManagementService);
67
- workflowService = await injector.resolveAsync(DocumentWorkflowService);
68
64
  categoryTypeService = await injector.resolveAsync(DocumentCategoryTypeService);
69
- propertyService = await injector.resolveAsync(DocumentPropertyService);
70
65
  });
71
66
  afterAll(async () => {
72
67
  await injector?.dispose();
@@ -7,7 +7,6 @@ import { clearTenantData, setupIntegrationTest } from '../../testing/index.js';
7
7
  import { DocumentManagementAuthorizationService } from '../authorization/document-management-authorization.service.js';
8
8
  import { DocumentManagementApiController } from '../server/api/document-management.api.js';
9
9
  import { configureDocumentManagement } from '../server/configure.js';
10
- import { migrateDocumentManagementSchema } from '../server/module.js';
11
10
  import { DocumentCollectionService } from '../server/services/document-collection.service.js';
12
11
  import { DocumentManagementAiService } from '../server/services/document-management-ai.service.js';
13
12
  import { DocumentService } from '../server/services/document.service.js';
@@ -22,7 +21,7 @@ describe('DocumentManagementApi Stats', () => {
22
21
  const tenantId = crypto.randomUUID();
23
22
  beforeAll(async () => {
24
23
  ({ injector, database } = await setupIntegrationTest({
25
- modules: { taskQueue: false, messageBus: true },
24
+ modules: { taskQueue: false, messageBus: true, documentManagement: true },
26
25
  orm: { schema },
27
26
  }));
28
27
  injector.register(GenkitModuleOptions, { useValue: {} });
@@ -50,8 +49,8 @@ describe('DocumentManagementApi Stats', () => {
50
49
  fileObjectStorageModule: 'documents',
51
50
  fileUploadObjectStorageModule: 'document-uploads',
52
51
  filePreviewObjectStorageModule: 'document-previews',
52
+ injector,
53
53
  });
54
- await runInInjectionContext(injector, migrateDocumentManagementSchema);
55
54
  controller = await injector.resolveAsync(DocumentManagementApiController);
56
55
  documentService = await injector.resolveAsync(DocumentService);
57
56
  collectionService = await injector.resolveAsync(DocumentCollectionService);
@@ -76,8 +75,8 @@ describe('DocumentManagementApi Stats', () => {
76
75
  parameters: {
77
76
  includeTotalCount: true,
78
77
  includeStorageUsage: true,
79
- collectionIds: [collection.id]
80
- }
78
+ collectionIds: [collection.id],
79
+ },
81
80
  };
82
81
  const stats = await controller.getStatistics(mockContext);
83
82
  expect(stats.totalCount).toBe(1);
@@ -93,8 +92,8 @@ describe('DocumentManagementApi Stats', () => {
93
92
  getToken: async () => ({ payload: { tenantId } }),
94
93
  parameters: {
95
94
  includeTotalCount: true,
96
- collectionIds: ['some-collection-id']
97
- }
95
+ collectionIds: ['some-collection-id'],
96
+ },
98
97
  };
99
98
  await expect(controller.getStatistics(mockContext)).rejects.toThrow('You are not allowed to read collection some-collection-id');
100
99
  });