@tstdl/base 0.93.139 → 0.93.140
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +166 -0
- package/ai/genkit/multi-region.plugin.js +5 -3
- package/ai/genkit/tests/multi-region.test.d.ts +1 -0
- package/ai/genkit/tests/multi-region.test.js +5 -2
- package/ai/parser/parser.js +2 -2
- package/ai/prompts/build.js +1 -0
- package/ai/prompts/instructions-formatter.d.ts +15 -2
- package/ai/prompts/instructions-formatter.js +36 -31
- package/ai/prompts/prompt-builder.js +5 -5
- package/ai/prompts/steering.d.ts +3 -2
- package/ai/prompts/steering.js +3 -1
- package/ai/tests/instructions-formatter.test.js +1 -0
- package/api/README.md +403 -0
- package/api/client/client.js +7 -13
- package/api/client/tests/api-client.test.js +10 -10
- package/api/default-error-handlers.js +1 -1
- package/api/response.d.ts +2 -2
- package/api/response.js +22 -33
- package/api/server/api-controller.d.ts +1 -1
- package/api/server/api-controller.js +3 -3
- package/api/server/api-request-token.provider.d.ts +1 -0
- package/api/server/api-request-token.provider.js +1 -0
- package/api/server/middlewares/allowed-methods.middleware.js +2 -1
- package/api/server/middlewares/content-type.middleware.js +2 -1
- package/api/types.d.ts +3 -2
- package/application/README.md +240 -0
- package/application/application.js +2 -2
- package/audit/README.md +267 -0
- package/authentication/README.md +288 -0
- package/authentication/client/authentication.service.d.ts +12 -11
- package/authentication/client/authentication.service.js +21 -21
- package/authentication/client/http-client.middleware.js +2 -2
- package/authentication/tests/authentication.client-error-handling.test.js +2 -1
- package/authentication/tests/authentication.client-service-refresh.test.js +5 -3
- package/browser/README.md +401 -0
- package/cancellation/README.md +156 -0
- package/cancellation/tests/coverage.test.d.ts +1 -0
- package/cancellation/tests/coverage.test.js +49 -0
- package/cancellation/tests/leak.test.js +24 -29
- package/cancellation/tests/token.test.d.ts +1 -0
- package/cancellation/tests/token.test.js +136 -0
- package/cancellation/token.d.ts +53 -177
- package/cancellation/token.js +132 -208
- package/context/README.md +174 -0
- package/cookie/README.md +161 -0
- package/css/README.md +157 -0
- package/data-structures/README.md +320 -0
- package/decorators/README.md +140 -0
- package/distributed-loop/README.md +231 -0
- package/distributed-loop/distributed-loop.js +1 -1
- package/document-management/README.md +403 -0
- package/document-management/server/services/document-management.service.js +9 -7
- package/document-management/tests/document-management-core.test.js +2 -7
- package/document-management/tests/document-management.api.test.js +6 -7
- package/document-management/tests/document-statistics.service.test.js +11 -12
- package/document-management/tests/document.service.test.js +3 -3
- package/document-management/tests/enum-helpers.test.js +2 -3
- package/dom/README.md +213 -0
- package/enumerable/README.md +259 -0
- package/enumeration/README.md +121 -0
- package/errors/README.md +267 -0
- package/file/README.md +191 -0
- package/formats/README.md +210 -0
- package/function/README.md +144 -0
- package/http/README.md +318 -0
- package/http/client/adapters/undici.adapter.js +1 -1
- package/http/client/http-client-request.d.ts +6 -5
- package/http/client/http-client-request.js +8 -9
- package/http/server/node/node-http-server.js +1 -2
- package/image-service/README.md +137 -0
- package/injector/README.md +491 -0
- package/intl/README.md +113 -0
- package/json-path/README.md +182 -0
- package/jsx/README.md +154 -0
- package/key-value-store/README.md +191 -0
- package/lock/README.md +249 -0
- package/lock/web/web-lock.js +119 -47
- package/logger/README.md +287 -0
- package/mail/README.md +256 -0
- package/memory/README.md +144 -0
- package/message-bus/README.md +244 -0
- package/message-bus/message-bus-base.js +1 -1
- package/module/README.md +182 -0
- package/module/module.d.ts +1 -1
- package/module/module.js +77 -17
- package/module/modules/web-server.module.js +1 -1
- package/notification/tests/notification-type.service.test.js +24 -15
- package/object-storage/README.md +300 -0
- package/openid-connect/README.md +274 -0
- package/orm/README.md +423 -0
- package/package.json +8 -6
- package/password/README.md +164 -0
- package/pdf/README.md +246 -0
- package/polyfills.js +1 -0
- package/pool/README.md +198 -0
- package/process/README.md +237 -0
- package/promise/README.md +252 -0
- package/promise/cancelable-promise.js +1 -1
- package/random/README.md +193 -0
- package/reflection/README.md +305 -0
- package/rpc/README.md +386 -0
- package/rxjs-utils/README.md +262 -0
- package/schema/README.md +342 -0
- package/serializer/README.md +342 -0
- package/signals/implementation/README.md +134 -0
- package/sse/README.md +278 -0
- package/task-queue/README.md +300 -0
- package/task-queue/postgres/task-queue.d.ts +2 -1
- package/task-queue/postgres/task-queue.js +32 -2
- package/task-queue/task-context.js +1 -1
- package/task-queue/task-queue.d.ts +17 -0
- package/task-queue/task-queue.js +103 -45
- package/task-queue/tests/complex.test.js +4 -4
- package/task-queue/tests/dependencies.test.js +4 -2
- package/task-queue/tests/queue.test.js +111 -0
- package/task-queue/tests/worker.test.js +21 -13
- package/templates/README.md +287 -0
- package/testing/README.md +157 -0
- package/text/README.md +346 -0
- package/threading/README.md +238 -0
- package/types/README.md +311 -0
- package/utils/README.md +322 -0
- package/utils/async-iterable-helpers/observable-iterable.d.ts +1 -1
- package/utils/async-iterable-helpers/observable-iterable.js +4 -8
- package/utils/async-iterable-helpers/take-until.js +4 -4
- package/utils/backoff.js +89 -30
- package/utils/retry-with-backoff.js +1 -1
- package/utils/timer.d.ts +1 -1
- package/utils/timer.js +5 -7
- package/utils/timing.d.ts +1 -1
- package/utils/timing.js +2 -4
- package/utils/z-base32.d.ts +1 -0
- package/utils/z-base32.js +1 -0
|
@@ -0,0 +1,403 @@
|
|
|
1
|
+
# Document Management Module
|
|
2
|
+
|
|
3
|
+
The `@tstdl/base/document-management` module provides a comprehensive, AI-enhanced system for managing document lifecycles. It features hierarchical collections, automated classification and data extraction, structured validation workflows, and a robust permission system.
|
|
4
|
+
|
|
5
|
+
## Table of Contents
|
|
6
|
+
|
|
7
|
+
- [✨ Features](#-features)
|
|
8
|
+
- [Core Concepts](#core-concepts)
|
|
9
|
+
- [🚀 Basic Usage](#-basic-usage)
|
|
10
|
+
- [Server Configuration](#server-configuration)
|
|
11
|
+
- [Client API Interaction](#client-api-interaction)
|
|
12
|
+
- [🔧 Advanced Topics](#-advanced-topics)
|
|
13
|
+
- [Defining Taxonomy (Categories & Types)](#defining-taxonomy-categories--types)
|
|
14
|
+
- [Implementing Authorization](#implementing-authorization)
|
|
15
|
+
- [Context Metadata (Ancillary Service)](#context-metadata-ancillary-service)
|
|
16
|
+
- [Advanced AI Configuration](#advanced-ai-configuration)
|
|
17
|
+
- [Document Requests & Templates](#document-requests--templates)
|
|
18
|
+
- [Retrieving Statistics](#retrieving-statistics)
|
|
19
|
+
- [Real-time Updates](#real-time-updates)
|
|
20
|
+
- [📚 API](#-api)
|
|
21
|
+
|
|
22
|
+
## ✨ Features
|
|
23
|
+
|
|
24
|
+
- **Hierarchical Collections**: Organize documents into logical groups (e.g., by tenant, case, or project) with support for infinitely nested sub-collections.
|
|
25
|
+
- **AI-Powered Workflow**: Asynchronous processing pipeline including:
|
|
26
|
+
- **Classification**: Automatically determines document type.
|
|
27
|
+
- **Extraction**: Extracts structured properties (dates, amounts, etc.) and tags.
|
|
28
|
+
- **Assignment**: Suggests target collections or requests based on content.
|
|
29
|
+
- **Human-in-the-Loop**: Multi-stage verification process. AI results are presented in "Review" states for manual verification and correction before proceeding.
|
|
30
|
+
- **Structured Taxonomy**: Define multi-level Categories, Document Types, and strongly-typed Properties (Text, Number, Date, Boolean).
|
|
31
|
+
- **Document Requests**: Formalize requirements for specific documents, track fulfillment, and use templates for recurring needs (e.g., onboarding packages).
|
|
32
|
+
- **Validation Framework**: Extensible validation logic integrated into the workflow. Support for automated checks and multi-document consistency.
|
|
33
|
+
- **Real-time Synchronization**: Built-in SSE support for live updates of document states, requests, and collection data.
|
|
34
|
+
- **Secure File Handling**: Direct-to-object-storage uploads via signed URLs and on-the-fly preview generation for various file types.
|
|
35
|
+
- **Comprehensive Statistics**: Built-in reporting on document counts, approval states, type distribution, and validation failures.
|
|
36
|
+
|
|
37
|
+
## Core Concepts
|
|
38
|
+
|
|
39
|
+
### Collections & Assignments
|
|
40
|
+
|
|
41
|
+
A **Collection** is the primary container for documents. Documents are not "stored" inside a collection physically but are **assigned** to them. A single document can be assigned to multiple collections.
|
|
42
|
+
|
|
43
|
+
### Taxonomy
|
|
44
|
+
|
|
45
|
+
The system relies on a structured definition of data:
|
|
46
|
+
|
|
47
|
+
1. **Categories**: High-level grouping (e.g., "Financial", "Legal"). Supports hierarchical nesting (sub-categories).
|
|
48
|
+
2. **Types**: Specific document kinds (e.g., "Invoice", "Contract") belonging to a category.
|
|
49
|
+
3. **Properties**: Typed data fields (e.g., "Invoice Amount", "Due Date") attached to specific Document Types.
|
|
50
|
+
|
|
51
|
+
### The Workflow & Immutability
|
|
52
|
+
|
|
53
|
+
Every uploaded document goes through a state machine managed by the `DocumentWorkflowService`:
|
|
54
|
+
|
|
55
|
+
1. **Classification**: AI determines the `DocumentType`.
|
|
56
|
+
2. **Extraction**: AI populates `DocumentProperties` and tags.
|
|
57
|
+
3. **Assignment**: AI suggests collections or requests if not explicitly provided during upload.
|
|
58
|
+
4. **Validation**: Runs automated checks (e.g., "Is this a valid ID card?").
|
|
59
|
+
|
|
60
|
+
Between steps, the document may enter a **Review** state (e.g., `Waiting for Review (Analysis)`), requiring a user to confirm or correct the data via the API (`proceedDocumentWorkflow`).
|
|
61
|
+
|
|
62
|
+
**Immutability**: Once a document is moved to the **Approved** state, it becomes read-only. This ensures data integrity for processed documents. Any errors discovered after approval require the document to be archived and a new version uploaded.
|
|
63
|
+
|
|
64
|
+
### Assignments & Archiving
|
|
65
|
+
|
|
66
|
+
Documents are linked to collections via **Assignments**. A document can be assigned to many collections simultaneously. If a document is no longer needed in a specific context, you can **archive** the assignment. This hides the document from that collection's views without deleting the document entity or affecting its assignments in other collections.
|
|
67
|
+
|
|
68
|
+
## 🚀 Basic Usage
|
|
69
|
+
|
|
70
|
+
### Server Configuration
|
|
71
|
+
|
|
72
|
+
To use the module, you must configure it during your application bootstrap. This involves registering the module and providing implementations for authorization and metadata resolution.
|
|
73
|
+
|
|
74
|
+
```typescript
|
|
75
|
+
import { configureDocumentManagement } from '@tstdl/base/document-management/server';
|
|
76
|
+
import { AppAncillaryService } from './services/app-ancillary.service.js';
|
|
77
|
+
import { AppAuthorizationService } from './services/app-authorization.service.js';
|
|
78
|
+
|
|
79
|
+
// In your application bootstrap function
|
|
80
|
+
export async function bootstrap() {
|
|
81
|
+
configureDocumentManagement({
|
|
82
|
+
// Service to resolve collection names/groups from your app's data
|
|
83
|
+
ancillaryService: AppAncillaryService,
|
|
84
|
+
|
|
85
|
+
// Service to handle permissions
|
|
86
|
+
authorizationService: AppAuthorizationService,
|
|
87
|
+
|
|
88
|
+
// Object Storage module names (must be configured via configureObjectStorage)
|
|
89
|
+
fileObjectStorageModule: 'documents',
|
|
90
|
+
fileUploadObjectStorageModule: 'document-uploads',
|
|
91
|
+
filePreviewObjectStorageModule: 'document-previews',
|
|
92
|
+
|
|
93
|
+
// Optional database config override
|
|
94
|
+
// database: { ... }
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
### Client API Interaction
|
|
100
|
+
|
|
101
|
+
The `DocumentManagementApi` provides a type-safe client for all operations.
|
|
102
|
+
|
|
103
|
+
#### 1. Uploading a Document
|
|
104
|
+
|
|
105
|
+
Uploading is a two-step process: get a signed URL, upload the binary, then create the entity.
|
|
106
|
+
|
|
107
|
+
```typescript
|
|
108
|
+
import { inject } from '@tstdl/base/injector';
|
|
109
|
+
import { DocumentManagementApi } from '@tstdl/base/document-management';
|
|
110
|
+
|
|
111
|
+
async function uploadInvoice(file: File, collectionId: string) {
|
|
112
|
+
const api = inject(DocumentManagementApi);
|
|
113
|
+
|
|
114
|
+
// 1. Initiate upload to get a secure URL
|
|
115
|
+
const { uploadId, uploadUrl } = await api.initiateDocumentUpload({
|
|
116
|
+
contentLength: file.size,
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
// 2. Upload the file content directly to storage
|
|
120
|
+
await fetch(uploadUrl, { method: 'PUT', body: file });
|
|
121
|
+
|
|
122
|
+
// 3. Create the document entity
|
|
123
|
+
const document = await api.createDocument({
|
|
124
|
+
uploadId,
|
|
125
|
+
originalFileName: file.name,
|
|
126
|
+
assignment: {
|
|
127
|
+
collections: [collectionId], // Assign to a specific collection
|
|
128
|
+
},
|
|
129
|
+
// Optional: Provide initial metadata if known
|
|
130
|
+
// title: 'My Invoice',
|
|
131
|
+
// typeId: 'invoice-type-id'
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
console.log('Document created:', document.id);
|
|
135
|
+
}
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
#### 2. Loading Data
|
|
139
|
+
|
|
140
|
+
Fetch all documents, requests, and metadata for a set of collections.
|
|
141
|
+
|
|
142
|
+
```typescript
|
|
143
|
+
import { inject } from '@tstdl/base/injector';
|
|
144
|
+
import { DocumentManagementApi, toEnrichedDocumentManagementData } from '@tstdl/base/document-management';
|
|
145
|
+
|
|
146
|
+
async function loadDashboard(collectionId: string) {
|
|
147
|
+
const api = inject(DocumentManagementApi);
|
|
148
|
+
|
|
149
|
+
// Load raw data
|
|
150
|
+
const rawData = await api.loadData({ collectionIds: [collectionId] });
|
|
151
|
+
|
|
152
|
+
// Convert to enriched view model for easier consumption (resolves references)
|
|
153
|
+
const data = toEnrichedDocumentManagementData(rawData);
|
|
154
|
+
|
|
155
|
+
for (const doc of data.documents) {
|
|
156
|
+
console.log(`Document: ${doc.title} (${doc.type?.label})`);
|
|
157
|
+
console.log(`Status: ${doc.approval}`);
|
|
158
|
+
|
|
159
|
+
// Access typed properties easily
|
|
160
|
+
const amount = doc.properties.find((p) => p.label === 'Amount')?.value;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
## 🔧 Advanced Topics
|
|
166
|
+
|
|
167
|
+
### Defining Taxonomy (Categories & Types)
|
|
168
|
+
|
|
169
|
+
You should seed your database with a defined taxonomy. This is typically done in a startup script or migration utility using `DocumentManagementService`.
|
|
170
|
+
|
|
171
|
+
```typescript
|
|
172
|
+
import { DocumentManagementService } from '@tstdl/base/document-management/server';
|
|
173
|
+
import { DocumentPropertyDataType } from '@tstdl/base/document-management';
|
|
174
|
+
import { defineEnum } from '@tstdl/base/enumeration';
|
|
175
|
+
import { inject } from '@tstdl/base/injector';
|
|
176
|
+
|
|
177
|
+
// Define Enums for type safety
|
|
178
|
+
const MyCategory = defineEnum('MyCategory', { Finance: 'finance' });
|
|
179
|
+
const MyType = defineEnum('MyType', { Invoice: 'invoice' });
|
|
180
|
+
const MyProp = defineEnum('MyProp', { Amount: 'amount', DueDate: 'due-date' });
|
|
181
|
+
|
|
182
|
+
async function seedTaxonomy(tenantId: string) {
|
|
183
|
+
const service = inject(DocumentManagementService);
|
|
184
|
+
|
|
185
|
+
await service.initializeCategoriesAndTypes(tenantId, {
|
|
186
|
+
categoryLabels: { [MyCategory.Finance]: 'Financial Documents' },
|
|
187
|
+
categoryParents: { [MyCategory.Finance]: null },
|
|
188
|
+
|
|
189
|
+
typeLabels: { [MyType.Invoice]: 'Incoming Invoice' },
|
|
190
|
+
typeCategories: { [MyType.Invoice]: MyCategory.Finance },
|
|
191
|
+
|
|
192
|
+
propertyConfigurations: {
|
|
193
|
+
[MyProp.Amount]: [DocumentPropertyDataType.Decimal, 'Total Amount'],
|
|
194
|
+
[MyProp.DueDate]: [DocumentPropertyDataType.Date, 'Payment Due Date'],
|
|
195
|
+
},
|
|
196
|
+
|
|
197
|
+
typeProperties: {
|
|
198
|
+
[MyType.Invoice]: [MyProp.Amount, MyProp.DueDate],
|
|
199
|
+
},
|
|
200
|
+
});
|
|
201
|
+
}
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
### Implementing Authorization
|
|
205
|
+
|
|
206
|
+
You must implement `DocumentManagementAuthorizationService` to control access. This service is called by the API layer.
|
|
207
|
+
|
|
208
|
+
```typescript
|
|
209
|
+
import { DocumentManagementAuthorizationService } from '@tstdl/base/document-management';
|
|
210
|
+
import { Singleton } from '@tstdl/base/injector';
|
|
211
|
+
|
|
212
|
+
// Assuming your app uses a specific token type
|
|
213
|
+
type MyToken = { payload: { userId: string; tenantId: string; roles: string[] } };
|
|
214
|
+
|
|
215
|
+
@Singleton()
|
|
216
|
+
export class AppAuthorizationService extends DocumentManagementAuthorizationService<MyToken> {
|
|
217
|
+
getTenantId(token?: MyToken): string {
|
|
218
|
+
return token?.payload.tenantId ?? '';
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
getSubject(token?: MyToken): string {
|
|
222
|
+
return token?.payload.userId ?? '';
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
async canReadCollection(collectionId: string, token?: MyToken): Promise<boolean> {
|
|
226
|
+
// Implement your logic: check if user has access to this collection
|
|
227
|
+
return true;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
async canApproveDocument(documentId: string, token?: MyToken): Promise<boolean> {
|
|
231
|
+
return token?.payload.roles.includes('manager') ?? false;
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
// ... implement other abstract methods (canCreateDocuments, canAssignDocuments, etc.)
|
|
235
|
+
}
|
|
236
|
+
```
|
|
237
|
+
|
|
238
|
+
### Context Metadata (Ancillary Service)
|
|
239
|
+
|
|
240
|
+
The `DocumentManagementAncillaryService` allows you to inject application-specific data (like names) into the document management views, as the module itself only knows about Collection IDs.
|
|
241
|
+
|
|
242
|
+
```typescript
|
|
243
|
+
import { DocumentManagementAncillaryService } from '@tstdl/base/document-management/server';
|
|
244
|
+
import { DocumentCollection, DocumentCollectionMetadata } from '@tstdl/base/document-management';
|
|
245
|
+
import { Singleton, inject } from '@tstdl/base/injector';
|
|
246
|
+
import { UserRepository } from './user.repository.js';
|
|
247
|
+
|
|
248
|
+
@Singleton()
|
|
249
|
+
export class AppAncillaryService extends DocumentManagementAncillaryService {
|
|
250
|
+
readonly #userRepo = inject(UserRepository);
|
|
251
|
+
|
|
252
|
+
async resolveMetadata(collections: DocumentCollection[]): Promise<DocumentCollectionMetadata[]> {
|
|
253
|
+
// Example: If collection.id corresponds to a User ID
|
|
254
|
+
const userIds = collections.map((c) => c.id);
|
|
255
|
+
const users = await this.#userRepo.loadMany(userIds);
|
|
256
|
+
|
|
257
|
+
return collections.map((collection) => {
|
|
258
|
+
const user = users.find((u) => u.id == collection.id);
|
|
259
|
+
return {
|
|
260
|
+
name: user ? `${user.firstName} ${user.lastName}` : 'Unknown Collection',
|
|
261
|
+
group: 'User Files',
|
|
262
|
+
};
|
|
263
|
+
});
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
```
|
|
267
|
+
|
|
268
|
+
### Advanced AI Configuration
|
|
269
|
+
|
|
270
|
+
The module allows for fine-grained control over AI behavior during classification and data extraction. Configuration follows a strict hierarchy:
|
|
271
|
+
|
|
272
|
+
1. **Document Type**: Config linked to a specific `DocumentType` key.
|
|
273
|
+
2. **Document Category**: Config linked to a specific `DocumentCategory` key.
|
|
274
|
+
3. **Step-Specific Global**: Global config for a `DocumentWorkflowStep`.
|
|
275
|
+
4. **Global Defaults**: Fallback module configuration.
|
|
276
|
+
|
|
277
|
+
#### Structured Overrides
|
|
278
|
+
|
|
279
|
+
For each core field (`title`, `subtitle`, `date`, `summary`, `tags`) and dynamic properties, you can provide granular instructions using the `InstructionOverride` type:
|
|
280
|
+
|
|
281
|
+
- **Simple Way**: Provide a `format` string (e.g., `INV-{{Date}}`).
|
|
282
|
+
- **Advanced Way**: Provide custom `content` and a `strategy` (`append` or `replace`).
|
|
283
|
+
|
|
284
|
+
```typescript
|
|
285
|
+
// Example configuration in your AI Provider
|
|
286
|
+
override getGlobalConfiguration() {
|
|
287
|
+
return {
|
|
288
|
+
documentTypes: {
|
|
289
|
+
'invoice': {
|
|
290
|
+
extraction: {
|
|
291
|
+
title: { format: 'Invoice - {{Vendor}} - {{Date}}' },
|
|
292
|
+
summary: { content: 'Focus on individual line items.', strategy: 'append' }
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
},
|
|
296
|
+
defaults: {
|
|
297
|
+
classification: { content: 'Always prefer finance categories if an IBAN is found.', strategy: 'append' }
|
|
298
|
+
}
|
|
299
|
+
};
|
|
300
|
+
}
|
|
301
|
+
```
|
|
302
|
+
|
|
303
|
+
### Document Requests & Templates
|
|
304
|
+
|
|
305
|
+
Create requests to ask users for specific documents.
|
|
306
|
+
|
|
307
|
+
```typescript
|
|
308
|
+
import { inject } from '@tstdl/base/injector';
|
|
309
|
+
import { DocumentManagementApi } from '@tstdl/base/document-management';
|
|
310
|
+
|
|
311
|
+
async function requestDocument() {
|
|
312
|
+
const api = inject(DocumentManagementApi);
|
|
313
|
+
|
|
314
|
+
// Create a single request
|
|
315
|
+
await api.createDocumentRequest({
|
|
316
|
+
collectionIds: ['target-collection-id'],
|
|
317
|
+
typeId: 'invoice-type-id',
|
|
318
|
+
comment: 'Please upload the invoice for March.',
|
|
319
|
+
});
|
|
320
|
+
|
|
321
|
+
// Or apply a pre-defined template
|
|
322
|
+
await api.applyDocumentRequestsTemplate({
|
|
323
|
+
id: 'onboarding-template-id',
|
|
324
|
+
collectionIds: ['new-employee-collection-id'],
|
|
325
|
+
});
|
|
326
|
+
}
|
|
327
|
+
```
|
|
328
|
+
|
|
329
|
+
### Retrieving Statistics
|
|
330
|
+
|
|
331
|
+
The module provides a flexible API to retrieve statistical insights about documents, such as counts, storage usage, and data quality metrics.
|
|
332
|
+
|
|
333
|
+
```typescript
|
|
334
|
+
import { inject } from '@tstdl/base/injector';
|
|
335
|
+
import { DocumentManagementApi } from '@tstdl/base/document-management';
|
|
336
|
+
|
|
337
|
+
async function getStats(collectionId: string) {
|
|
338
|
+
const api = inject(DocumentManagementApi);
|
|
339
|
+
|
|
340
|
+
const stats = await api.getStatistics({
|
|
341
|
+
includeTotalCount: true,
|
|
342
|
+
includeApprovalBreakdown: true,
|
|
343
|
+
includeStorageUsage: true,
|
|
344
|
+
includeDataQuality: true,
|
|
345
|
+
collectionIds: [collectionId], // Optional: Filter by specific collections
|
|
346
|
+
});
|
|
347
|
+
|
|
348
|
+
console.log(`Total Documents: ${stats.totalCount}`);
|
|
349
|
+
console.log(`Storage Used: ${stats.storageUsage} bytes`);
|
|
350
|
+
console.log(`Approved: ${stats.approvalBreakdown?.approved}`);
|
|
351
|
+
console.log(`Documents missing type: ${stats.dataQuality?.missingType}`);
|
|
352
|
+
}
|
|
353
|
+
```
|
|
354
|
+
|
|
355
|
+
### Real-time Updates
|
|
356
|
+
|
|
357
|
+
The module supports Server-Sent Events (SSE) to push updates to the client when documents or collections change.
|
|
358
|
+
|
|
359
|
+
```typescript
|
|
360
|
+
import { inject } from '@tstdl/base/injector';
|
|
361
|
+
import { DocumentManagementApi } from '@tstdl/base/document-management';
|
|
362
|
+
|
|
363
|
+
async function watchCollection(collectionId: string) {
|
|
364
|
+
const api = inject(DocumentManagementApi);
|
|
365
|
+
|
|
366
|
+
// Returns an AsyncIterable that yields new data whenever changes occur
|
|
367
|
+
const stream = api.loadDataStream({ collectionIds: [collectionId] });
|
|
368
|
+
|
|
369
|
+
for await (const data of stream) {
|
|
370
|
+
console.log('Data updated:', data);
|
|
371
|
+
// Update UI...
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
```
|
|
375
|
+
|
|
376
|
+
## 📚 API
|
|
377
|
+
|
|
378
|
+
### Client / Shared
|
|
379
|
+
|
|
380
|
+
| Export | Description |
|
|
381
|
+
| :--------------------------------------- | :------------------------------------------------------------------------------ |
|
|
382
|
+
| `DocumentManagementApi` | Main client class for API interaction. |
|
|
383
|
+
| `DocumentManagementData` | DTO containing all loaded data (collections, documents, types, etc.). |
|
|
384
|
+
| `toEnrichedDocumentManagementData` | Helper to convert raw DTO into an object graph with resolved references. |
|
|
385
|
+
| `DocumentView` | Represents a document with its properties, tags, and workflow status. |
|
|
386
|
+
| `DocumentRequestView` | Represents a request for a document. |
|
|
387
|
+
| `DocumentPropertyDataType` | Enum defining supported property types (Text, Integer, Decimal, Boolean, Date). |
|
|
388
|
+
| `DocumentManagementAuthorizationService` | Abstract base class for authorization logic. |
|
|
389
|
+
|
|
390
|
+
### Server
|
|
391
|
+
|
|
392
|
+
| Export | Description |
|
|
393
|
+
| :----------------------------------- | :------------------------------------------------------------- |
|
|
394
|
+
| `configureDocumentManagement` | Function to configure the module in the application bootstrap. |
|
|
395
|
+
| `DocumentManagementService` | Core service for managing data and initializing taxonomy. |
|
|
396
|
+
| `DocumentService` | Service for creating and updating documents. |
|
|
397
|
+
| `DocumentFileService` | Service for handling file uploads, storage, and previews. |
|
|
398
|
+
| `DocumentRequestService` | Service for managing document requests and templates. |
|
|
399
|
+
| `DocumentStatisticsService` | Service for retrieving document statistics. |
|
|
400
|
+
| `DocumentWorkflowService` | Service for managing the document processing workflow. |
|
|
401
|
+
| `DocumentManagementAncillaryService` | Abstract base class for resolving collection metadata. |
|
|
402
|
+
| `DocumentValidationExecutor` | Abstract base class for implementing custom validation logic. |
|
|
403
|
+
| `AiValidationExecutor` | Base class for AI-driven validation executors. |
|
|
@@ -8,18 +8,17 @@ var _a;
|
|
|
8
8
|
var DocumentManagementService_1;
|
|
9
9
|
import { and, eq } from 'drizzle-orm';
|
|
10
10
|
import { union } from 'drizzle-orm/pg-core';
|
|
11
|
+
import { filter, firstValueFrom, race, share, takeUntil } from 'rxjs';
|
|
11
12
|
import { Enumerable } from '../../../enumerable/index.js';
|
|
12
13
|
import { BadRequestError } from '../../../errors/bad-request.error.js';
|
|
13
14
|
import { inject } from '../../../injector/index.js';
|
|
14
15
|
import { Logger } from '../../../logger/logger.js';
|
|
15
16
|
import { Transactional, injectRepository, injectTransactional } from '../../../orm/server/index.js';
|
|
16
|
-
import { DeferredPromise } from '../../../promise/deferred-promise.js';
|
|
17
17
|
import { distinct } from '../../../utils/array/index.js';
|
|
18
18
|
import { compareByValueSelection } from '../../../utils/comparison.js';
|
|
19
19
|
import { groupToMap, groupToSingleMap } from '../../../utils/iterable-helpers/index.js';
|
|
20
20
|
import { fromEntries, objectEntries } from '../../../utils/object/index.js';
|
|
21
21
|
import { assertDefinedPass, isDefined, isNotNull, isNull, isUndefined } from '../../../utils/type-guards.js';
|
|
22
|
-
import { filter, merge } from 'rxjs';
|
|
23
22
|
import { DocumentAssignmentScope, DocumentAssignmentTask, DocumentCategory, DocumentCollectionAssignment, DocumentRequest, DocumentRequestCollectionAssignment, DocumentRequestTemplate, DocumentRequestsTemplate, DocumentType, DocumentTypeProperty, DocumentValidationExecution, DocumentWorkflowStep } from '../../models/index.js';
|
|
24
23
|
import { documentAssignmentScope, documentAssignmentTask, documentCollectionAssignment, documentRequest, documentRequestCollectionAssignment } from '../schemas.js';
|
|
25
24
|
import { DocumentCategoryTypeService } from './document-category-type.service.js';
|
|
@@ -72,14 +71,17 @@ let DocumentManagementService = DocumentManagementService_1 = class DocumentMana
|
|
|
72
71
|
return result.map((row) => row.collectionId);
|
|
73
72
|
}
|
|
74
73
|
async *loadDataStream(tenantId, collectionIds, cancellationSignal) {
|
|
75
|
-
const
|
|
76
|
-
|
|
77
|
-
const
|
|
74
|
+
const collectionIdsSet = new Set(collectionIds);
|
|
75
|
+
let hasNewData = false;
|
|
76
|
+
const newData$ = this.#observationService.collectionsChangedMessageBus.allMessages$.pipe(filter((changedCollectionIds) => changedCollectionIds.some((id) => collectionIdsSet.has(id))), takeUntil(cancellationSignal), share());
|
|
77
|
+
const subscription = newData$.subscribe(() => hasNewData = true);
|
|
78
78
|
try {
|
|
79
79
|
while (cancellationSignal.isUnset) {
|
|
80
|
+
hasNewData = false;
|
|
80
81
|
yield await this.loadData(tenantId, collectionIds);
|
|
81
|
-
|
|
82
|
-
|
|
82
|
+
if (!hasNewData && cancellationSignal.isUnset) {
|
|
83
|
+
await firstValueFrom(race(cancellationSignal, newData$));
|
|
84
|
+
}
|
|
83
85
|
}
|
|
84
86
|
}
|
|
85
87
|
finally {
|
|
@@ -5,7 +5,6 @@ import { ObjectStorage } from '../../object-storage/index.js';
|
|
|
5
5
|
import { TaskQueue } from '../../task-queue/index.js';
|
|
6
6
|
import { clearTenantData, setupIntegrationTest } from '../../testing/index.js';
|
|
7
7
|
import { configureDocumentManagement } from '../server/configure.js';
|
|
8
|
-
import { migrateDocumentManagementSchema } from '../server/module.js';
|
|
9
8
|
import { DocumentCategoryTypeService } from '../server/services/document-category-type.service.js';
|
|
10
9
|
import { DocumentCollectionService } from '../server/services/document-collection.service.js';
|
|
11
10
|
import { DocumentManagementAiService } from '../server/services/document-management-ai.service.js';
|
|
@@ -23,13 +22,11 @@ describe('Document Management Core', () => {
|
|
|
23
22
|
let requestService;
|
|
24
23
|
let managementService;
|
|
25
24
|
let categoryTypeService;
|
|
26
|
-
let workflowService;
|
|
27
|
-
let propertyService;
|
|
28
25
|
const schema = 'document_management';
|
|
29
26
|
const tenantId = crypto.randomUUID();
|
|
30
27
|
beforeAll(async () => {
|
|
31
28
|
({ injector, database } = await setupIntegrationTest({
|
|
32
|
-
modules: { taskQueue: false, messageBus: true }, // Disabled TaskQueue to avoid background noise
|
|
29
|
+
modules: { taskQueue: false, messageBus: true, documentManagement: true }, // Disabled TaskQueue to avoid background noise
|
|
33
30
|
orm: { schema },
|
|
34
31
|
}));
|
|
35
32
|
injector.register(GenkitModuleOptions, { useValue: {} });
|
|
@@ -58,15 +55,13 @@ describe('Document Management Core', () => {
|
|
|
58
55
|
fileObjectStorageModule: 'documents',
|
|
59
56
|
fileUploadObjectStorageModule: 'document-uploads',
|
|
60
57
|
filePreviewObjectStorageModule: 'document-previews',
|
|
58
|
+
injector,
|
|
61
59
|
});
|
|
62
|
-
await runInInjectionContext(injector, migrateDocumentManagementSchema);
|
|
63
60
|
documentService = await injector.resolveAsync(DocumentService);
|
|
64
61
|
collectionService = await injector.resolveAsync(DocumentCollectionService);
|
|
65
62
|
requestService = await injector.resolveAsync(DocumentRequestService);
|
|
66
63
|
managementService = await injector.resolveAsync(DocumentManagementService);
|
|
67
|
-
workflowService = await injector.resolveAsync(DocumentWorkflowService);
|
|
68
64
|
categoryTypeService = await injector.resolveAsync(DocumentCategoryTypeService);
|
|
69
|
-
propertyService = await injector.resolveAsync(DocumentPropertyService);
|
|
70
65
|
});
|
|
71
66
|
afterAll(async () => {
|
|
72
67
|
await injector?.dispose();
|
|
@@ -7,7 +7,6 @@ import { clearTenantData, setupIntegrationTest } from '../../testing/index.js';
|
|
|
7
7
|
import { DocumentManagementAuthorizationService } from '../authorization/document-management-authorization.service.js';
|
|
8
8
|
import { DocumentManagementApiController } from '../server/api/document-management.api.js';
|
|
9
9
|
import { configureDocumentManagement } from '../server/configure.js';
|
|
10
|
-
import { migrateDocumentManagementSchema } from '../server/module.js';
|
|
11
10
|
import { DocumentCollectionService } from '../server/services/document-collection.service.js';
|
|
12
11
|
import { DocumentManagementAiService } from '../server/services/document-management-ai.service.js';
|
|
13
12
|
import { DocumentService } from '../server/services/document.service.js';
|
|
@@ -22,7 +21,7 @@ describe('DocumentManagementApi Stats', () => {
|
|
|
22
21
|
const tenantId = crypto.randomUUID();
|
|
23
22
|
beforeAll(async () => {
|
|
24
23
|
({ injector, database } = await setupIntegrationTest({
|
|
25
|
-
modules: { taskQueue: false, messageBus: true },
|
|
24
|
+
modules: { taskQueue: false, messageBus: true, documentManagement: true },
|
|
26
25
|
orm: { schema },
|
|
27
26
|
}));
|
|
28
27
|
injector.register(GenkitModuleOptions, { useValue: {} });
|
|
@@ -50,8 +49,8 @@ describe('DocumentManagementApi Stats', () => {
|
|
|
50
49
|
fileObjectStorageModule: 'documents',
|
|
51
50
|
fileUploadObjectStorageModule: 'document-uploads',
|
|
52
51
|
filePreviewObjectStorageModule: 'document-previews',
|
|
52
|
+
injector,
|
|
53
53
|
});
|
|
54
|
-
await runInInjectionContext(injector, migrateDocumentManagementSchema);
|
|
55
54
|
controller = await injector.resolveAsync(DocumentManagementApiController);
|
|
56
55
|
documentService = await injector.resolveAsync(DocumentService);
|
|
57
56
|
collectionService = await injector.resolveAsync(DocumentCollectionService);
|
|
@@ -76,8 +75,8 @@ describe('DocumentManagementApi Stats', () => {
|
|
|
76
75
|
parameters: {
|
|
77
76
|
includeTotalCount: true,
|
|
78
77
|
includeStorageUsage: true,
|
|
79
|
-
collectionIds: [collection.id]
|
|
80
|
-
}
|
|
78
|
+
collectionIds: [collection.id],
|
|
79
|
+
},
|
|
81
80
|
};
|
|
82
81
|
const stats = await controller.getStatistics(mockContext);
|
|
83
82
|
expect(stats.totalCount).toBe(1);
|
|
@@ -93,8 +92,8 @@ describe('DocumentManagementApi Stats', () => {
|
|
|
93
92
|
getToken: async () => ({ payload: { tenantId } }),
|
|
94
93
|
parameters: {
|
|
95
94
|
includeTotalCount: true,
|
|
96
|
-
collectionIds: ['some-collection-id']
|
|
97
|
-
}
|
|
95
|
+
collectionIds: ['some-collection-id'],
|
|
96
|
+
},
|
|
98
97
|
};
|
|
99
98
|
await expect(controller.getStatistics(mockContext)).rejects.toThrow('You are not allowed to read collection some-collection-id');
|
|
100
99
|
});
|
|
@@ -9,7 +9,6 @@ import { DocumentValidationExecution, DocumentValidationExecutionState, Document
|
|
|
9
9
|
import { DocumentWorkflow, DocumentWorkflowState, DocumentWorkflowStep } from '../models/document-workflow.model.js';
|
|
10
10
|
import { DocumentApproval } from '../models/document.model.js';
|
|
11
11
|
import { configureDocumentManagement } from '../server/configure.js';
|
|
12
|
-
import { migrateDocumentManagementSchema } from '../server/module.js';
|
|
13
12
|
import { DocumentCategoryTypeService } from '../server/services/document-category-type.service.js';
|
|
14
13
|
import { DocumentCollectionService } from '../server/services/document-collection.service.js';
|
|
15
14
|
import { DocumentManagementAiService } from '../server/services/document-management-ai.service.js';
|
|
@@ -29,7 +28,7 @@ describe('DocumentStatisticsService', () => {
|
|
|
29
28
|
const tenantId = crypto.randomUUID();
|
|
30
29
|
beforeAll(async () => {
|
|
31
30
|
({ injector, database } = await setupIntegrationTest({
|
|
32
|
-
modules: { taskQueue: true },
|
|
31
|
+
modules: { taskQueue: true, documentManagement: true },
|
|
33
32
|
orm: { schema },
|
|
34
33
|
}));
|
|
35
34
|
const mockObjectStorage = {
|
|
@@ -56,8 +55,8 @@ describe('DocumentStatisticsService', () => {
|
|
|
56
55
|
fileObjectStorageModule: 'documents',
|
|
57
56
|
fileUploadObjectStorageModule: 'document-uploads',
|
|
58
57
|
filePreviewObjectStorageModule: 'document-previews',
|
|
58
|
+
injector,
|
|
59
59
|
});
|
|
60
|
-
await runInInjectionContext(injector, migrateDocumentManagementSchema);
|
|
61
60
|
documentService = await injector.resolveAsync(DocumentService);
|
|
62
61
|
statsService = await injector.resolveAsync(DocumentStatisticsService);
|
|
63
62
|
collectionService = await injector.resolveAsync(DocumentCollectionService);
|
|
@@ -288,7 +287,7 @@ describe('DocumentStatisticsService', () => {
|
|
|
288
287
|
const stats = await statsService.getStatistics(tenantId, {
|
|
289
288
|
includeTotalCount: true,
|
|
290
289
|
collectionIds: [collection1.id],
|
|
291
|
-
categoryIds: [category1.id]
|
|
290
|
+
categoryIds: [category1.id],
|
|
292
291
|
});
|
|
293
292
|
expect(stats.totalCount).toBe(1); // Only Doc 1 matches both
|
|
294
293
|
});
|
|
@@ -310,7 +309,7 @@ describe('DocumentStatisticsService', () => {
|
|
|
310
309
|
includeMimeTypeDistribution: true,
|
|
311
310
|
includeStorageUsage: true,
|
|
312
311
|
includeDataQuality: true,
|
|
313
|
-
collectionIds: [collection.id]
|
|
312
|
+
collectionIds: [collection.id],
|
|
314
313
|
});
|
|
315
314
|
expect(stats.totalCount).toBe(1);
|
|
316
315
|
expect(stats.typeBreakdown?.[type.id]).toBe(1);
|
|
@@ -328,7 +327,7 @@ describe('DocumentStatisticsService', () => {
|
|
|
328
327
|
const stats = await statsService.getStatistics(tenantId, {
|
|
329
328
|
includeTotalCount: true,
|
|
330
329
|
includeMimeTypeDistribution: true,
|
|
331
|
-
collectionIds: [collection.id]
|
|
330
|
+
collectionIds: [collection.id],
|
|
332
331
|
});
|
|
333
332
|
expect(stats.totalCount).toBe(1);
|
|
334
333
|
});
|
|
@@ -346,7 +345,7 @@ describe('DocumentStatisticsService', () => {
|
|
|
346
345
|
const stats = await statsService.getStatistics(tenantId, {
|
|
347
346
|
includeMimeTypeDistribution: true,
|
|
348
347
|
collectionIds: [collection.id],
|
|
349
|
-
categoryIds: [category.id]
|
|
348
|
+
categoryIds: [category.id],
|
|
350
349
|
});
|
|
351
350
|
expect(Object.keys(stats.mimeTypeDistribution ?? {}).length).toBeGreaterThan(0);
|
|
352
351
|
});
|
|
@@ -366,7 +365,7 @@ describe('DocumentStatisticsService', () => {
|
|
|
366
365
|
const stats = await statsService.getStatistics(tenantId, {
|
|
367
366
|
includeTagUsage: true,
|
|
368
367
|
collectionIds: [collection.id],
|
|
369
|
-
categoryIds: [category.id]
|
|
368
|
+
categoryIds: [category.id],
|
|
370
369
|
});
|
|
371
370
|
expect(stats.tagUsage?.[tag.id]).toBe(1);
|
|
372
371
|
});
|
|
@@ -410,7 +409,7 @@ describe('DocumentStatisticsService', () => {
|
|
|
410
409
|
});
|
|
411
410
|
const stats = await statsService.getStatistics(tenantId, {
|
|
412
411
|
includeValidationFailures: true,
|
|
413
|
-
collectionIds: [collection.id]
|
|
412
|
+
collectionIds: [collection.id],
|
|
414
413
|
});
|
|
415
414
|
expect(stats.validationFailures).toBe(1);
|
|
416
415
|
});
|
|
@@ -459,7 +458,7 @@ describe('DocumentStatisticsService', () => {
|
|
|
459
458
|
const stats = await statsService.getStatistics(tenantId, {
|
|
460
459
|
includeValidationFailures: true,
|
|
461
460
|
collectionIds: [collection.id],
|
|
462
|
-
categoryIds: [category.id]
|
|
461
|
+
categoryIds: [category.id],
|
|
463
462
|
});
|
|
464
463
|
expect(stats.validationFailures).toBe(1);
|
|
465
464
|
});
|
|
@@ -476,7 +475,7 @@ describe('DocumentStatisticsService', () => {
|
|
|
476
475
|
}, new Uint8Array([1]));
|
|
477
476
|
const stats = await statsService.getStatistics(tenantId, {
|
|
478
477
|
includeTypeBreakdown: true,
|
|
479
|
-
collectionIds: [collection.id]
|
|
478
|
+
collectionIds: [collection.id],
|
|
480
479
|
});
|
|
481
480
|
expect(stats.typeBreakdown?.[type.id]).toBe(1);
|
|
482
481
|
});
|
|
@@ -488,7 +487,7 @@ describe('DocumentStatisticsService', () => {
|
|
|
488
487
|
includeStorageUsage: true,
|
|
489
488
|
includeTotalPages: true,
|
|
490
489
|
includeValidationFailures: true,
|
|
491
|
-
collectionIds: []
|
|
490
|
+
collectionIds: [],
|
|
492
491
|
});
|
|
493
492
|
expect(stats.totalCount).toBe(0);
|
|
494
493
|
expect(stats.storageUsage).toBe(0);
|
|
@@ -6,7 +6,7 @@ import { clearTenantData, setupIntegrationTest } from '../../testing/index.js';
|
|
|
6
6
|
import { DocumentWorkflowState, DocumentWorkflowStep } from '../models/document-workflow.model.js';
|
|
7
7
|
import { DocumentApproval } from '../models/document.model.js';
|
|
8
8
|
import { configureDocumentManagement } from '../server/configure.js';
|
|
9
|
-
import { DocumentManagementConfiguration
|
|
9
|
+
import { DocumentManagementConfiguration } from '../server/module.js';
|
|
10
10
|
import { DocumentCollectionService } from '../server/services/document-collection.service.js';
|
|
11
11
|
import { DocumentManagementAiService } from '../server/services/document-management-ai.service.js';
|
|
12
12
|
import { DocumentWorkflowService } from '../server/services/document-workflow.service.js';
|
|
@@ -22,7 +22,7 @@ describe('DocumentService', () => {
|
|
|
22
22
|
const tenantId = crypto.randomUUID();
|
|
23
23
|
beforeAll(async () => {
|
|
24
24
|
({ injector, database } = await setupIntegrationTest({
|
|
25
|
-
modules: { taskQueue: true },
|
|
25
|
+
modules: { taskQueue: true, documentManagement: true },
|
|
26
26
|
orm: { schema },
|
|
27
27
|
}));
|
|
28
28
|
const mockObjectStorage = {
|
|
@@ -49,8 +49,8 @@ describe('DocumentService', () => {
|
|
|
49
49
|
fileObjectStorageModule: 'documents',
|
|
50
50
|
fileUploadObjectStorageModule: 'document-uploads',
|
|
51
51
|
filePreviewObjectStorageModule: 'document-previews',
|
|
52
|
+
injector,
|
|
52
53
|
});
|
|
53
|
-
await runInInjectionContext(injector, migrateDocumentManagementSchema);
|
|
54
54
|
documentService = await injector.resolveAsync(DocumentService);
|
|
55
55
|
workflowService = await injector.resolveAsync(DocumentWorkflowService);
|
|
56
56
|
collectionService = await injector.resolveAsync(DocumentCollectionService);
|