@langchain/core 0.1.63 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/callbacks/base.cjs +9 -1
- package/dist/callbacks/base.d.ts +3 -0
- package/dist/callbacks/base.js +9 -1
- package/dist/callbacks/manager.cjs +51 -0
- package/dist/callbacks/manager.js +51 -0
- package/dist/document_loaders/base.cjs +24 -0
- package/dist/document_loaders/base.d.ts +28 -0
- package/dist/document_loaders/base.js +20 -0
- package/dist/documents/transformers.d.ts +1 -1
- package/dist/indexing/base.cjs +270 -0
- package/dist/indexing/base.d.ts +114 -0
- package/dist/indexing/base.js +261 -0
- package/dist/indexing/index.cjs +18 -0
- package/dist/indexing/index.d.ts +2 -0
- package/dist/indexing/index.js +2 -0
- package/dist/indexing/record_manager.cjs +18 -0
- package/dist/indexing/record_manager.d.ts +64 -0
- package/dist/indexing/record_manager.js +14 -0
- package/dist/language_models/chat_models.cjs +21 -3
- package/dist/language_models/chat_models.d.ts +9 -0
- package/dist/language_models/chat_models.js +21 -3
- package/dist/runnables/base.cjs +39 -14
- package/dist/runnables/base.d.ts +4 -4
- package/dist/runnables/base.js +39 -14
- package/dist/runnables/passthrough.cjs +1 -0
- package/dist/runnables/passthrough.d.ts +1 -1
- package/dist/runnables/passthrough.js +1 -0
- package/dist/runnables/remote.cjs +60 -48
- package/dist/runnables/remote.d.ts +6 -2
- package/dist/runnables/remote.js +61 -49
- package/dist/utils/stream.cjs +27 -11
- package/dist/utils/stream.d.ts +6 -1
- package/dist/utils/stream.js +27 -11
- package/document_loaders/base.cjs +1 -0
- package/document_loaders/base.d.cts +1 -0
- package/document_loaders/base.d.ts +1 -0
- package/document_loaders/base.js +1 -0
- package/indexing.cjs +1 -0
- package/indexing.d.cts +1 -0
- package/indexing.d.ts +1 -0
- package/indexing.js +1 -0
- package/package.json +27 -1
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.index = exports._isBaseDocumentLoader = exports._getSourceIdAssigner = exports._deduplicateInOrder = exports._batch = exports._HashedDocument = void 0;
|
|
4
|
+
const uuid_1 = require("uuid");
|
|
5
|
+
const record_manager_js_1 = require("./record_manager.cjs");
|
|
6
|
+
const hash_js_1 = require("../utils/hash.cjs");
|
|
7
|
+
const document_js_1 = require("../documents/document.cjs");
|
|
8
|
+
/**
|
|
9
|
+
* HashedDocument is a Document with hashes calculated.
|
|
10
|
+
* Hashes are calculated based on page content and metadata.
|
|
11
|
+
* It is used for indexing.
|
|
12
|
+
*/
|
|
13
|
+
class _HashedDocument {
|
|
14
|
+
constructor(fields) {
|
|
15
|
+
Object.defineProperty(this, "uid", {
|
|
16
|
+
enumerable: true,
|
|
17
|
+
configurable: true,
|
|
18
|
+
writable: true,
|
|
19
|
+
value: void 0
|
|
20
|
+
});
|
|
21
|
+
Object.defineProperty(this, "hash_", {
|
|
22
|
+
enumerable: true,
|
|
23
|
+
configurable: true,
|
|
24
|
+
writable: true,
|
|
25
|
+
value: void 0
|
|
26
|
+
});
|
|
27
|
+
Object.defineProperty(this, "contentHash", {
|
|
28
|
+
enumerable: true,
|
|
29
|
+
configurable: true,
|
|
30
|
+
writable: true,
|
|
31
|
+
value: void 0
|
|
32
|
+
});
|
|
33
|
+
Object.defineProperty(this, "metadataHash", {
|
|
34
|
+
enumerable: true,
|
|
35
|
+
configurable: true,
|
|
36
|
+
writable: true,
|
|
37
|
+
value: void 0
|
|
38
|
+
});
|
|
39
|
+
Object.defineProperty(this, "pageContent", {
|
|
40
|
+
enumerable: true,
|
|
41
|
+
configurable: true,
|
|
42
|
+
writable: true,
|
|
43
|
+
value: void 0
|
|
44
|
+
});
|
|
45
|
+
Object.defineProperty(this, "metadata", {
|
|
46
|
+
enumerable: true,
|
|
47
|
+
configurable: true,
|
|
48
|
+
writable: true,
|
|
49
|
+
value: void 0
|
|
50
|
+
});
|
|
51
|
+
this.uid = fields.uid;
|
|
52
|
+
this.pageContent = fields.pageContent;
|
|
53
|
+
this.metadata = fields.metadata;
|
|
54
|
+
}
|
|
55
|
+
calculateHashes() {
|
|
56
|
+
const forbiddenKeys = ["hash_", "content_hash", "metadata_hash"];
|
|
57
|
+
for (const key of forbiddenKeys) {
|
|
58
|
+
if (key in this.metadata) {
|
|
59
|
+
throw new Error(`Metadata cannot contain key ${key} as it is reserved for internal use. Restricted keys: [${forbiddenKeys.join(", ")}]`);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
const contentHash = this._hashStringToUUID(this.pageContent);
|
|
63
|
+
try {
|
|
64
|
+
const metadataHash = this._hashNestedDictToUUID(this.metadata);
|
|
65
|
+
this.contentHash = contentHash;
|
|
66
|
+
this.metadataHash = metadataHash;
|
|
67
|
+
}
|
|
68
|
+
catch (e) {
|
|
69
|
+
throw new Error(`Failed to hash metadata: ${e}. Please use a dict that can be serialized using json.`);
|
|
70
|
+
}
|
|
71
|
+
this.hash_ = this._hashStringToUUID(this.contentHash + this.metadataHash);
|
|
72
|
+
if (!this.uid) {
|
|
73
|
+
this.uid = this.hash_;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
toDocument() {
|
|
77
|
+
return new document_js_1.Document({
|
|
78
|
+
pageContent: this.pageContent,
|
|
79
|
+
metadata: this.metadata,
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
static fromDocument(document, uid) {
|
|
83
|
+
const doc = new this({
|
|
84
|
+
pageContent: document.pageContent,
|
|
85
|
+
metadata: document.metadata,
|
|
86
|
+
uid: uid || document.uid,
|
|
87
|
+
});
|
|
88
|
+
doc.calculateHashes();
|
|
89
|
+
return doc;
|
|
90
|
+
}
|
|
91
|
+
_hashStringToUUID(inputString) {
|
|
92
|
+
const hash_value = (0, hash_js_1.insecureHash)(inputString);
|
|
93
|
+
return (0, uuid_1.v5)(hash_value, record_manager_js_1.UUIDV5_NAMESPACE);
|
|
94
|
+
}
|
|
95
|
+
_hashNestedDictToUUID(data) {
|
|
96
|
+
const serialized_data = JSON.stringify(data, Object.keys(data).sort());
|
|
97
|
+
const hash_value = (0, hash_js_1.insecureHash)(serialized_data);
|
|
98
|
+
return (0, uuid_1.v5)(hash_value, record_manager_js_1.UUIDV5_NAMESPACE);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
exports._HashedDocument = _HashedDocument;
|
|
102
|
+
function _batch(size, iterable) {
|
|
103
|
+
const batches = [];
|
|
104
|
+
let currentBatch = [];
|
|
105
|
+
iterable.forEach((item) => {
|
|
106
|
+
currentBatch.push(item);
|
|
107
|
+
if (currentBatch.length >= size) {
|
|
108
|
+
batches.push(currentBatch);
|
|
109
|
+
currentBatch = [];
|
|
110
|
+
}
|
|
111
|
+
});
|
|
112
|
+
if (currentBatch.length > 0) {
|
|
113
|
+
batches.push(currentBatch);
|
|
114
|
+
}
|
|
115
|
+
return batches;
|
|
116
|
+
}
|
|
117
|
+
exports._batch = _batch;
|
|
118
|
+
function _deduplicateInOrder(hashedDocuments) {
|
|
119
|
+
const seen = new Set();
|
|
120
|
+
const deduplicated = [];
|
|
121
|
+
for (const hashedDoc of hashedDocuments) {
|
|
122
|
+
if (!hashedDoc.hash_) {
|
|
123
|
+
throw new Error("Hashed document does not have a hash");
|
|
124
|
+
}
|
|
125
|
+
if (!seen.has(hashedDoc.hash_)) {
|
|
126
|
+
seen.add(hashedDoc.hash_);
|
|
127
|
+
deduplicated.push(hashedDoc);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
return deduplicated;
|
|
131
|
+
}
|
|
132
|
+
exports._deduplicateInOrder = _deduplicateInOrder;
|
|
133
|
+
function _getSourceIdAssigner(sourceIdKey) {
|
|
134
|
+
if (sourceIdKey === null) {
|
|
135
|
+
return (_doc) => null;
|
|
136
|
+
}
|
|
137
|
+
else if (typeof sourceIdKey === "string") {
|
|
138
|
+
return (doc) => doc.metadata[sourceIdKey];
|
|
139
|
+
}
|
|
140
|
+
else if (typeof sourceIdKey === "function") {
|
|
141
|
+
return sourceIdKey;
|
|
142
|
+
}
|
|
143
|
+
else {
|
|
144
|
+
throw new Error(`sourceIdKey should be null, a string or a function, got ${typeof sourceIdKey}`);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
exports._getSourceIdAssigner = _getSourceIdAssigner;
|
|
148
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
149
|
+
const _isBaseDocumentLoader = (arg) => {
|
|
150
|
+
if ("load" in arg &&
|
|
151
|
+
typeof arg.load === "function" &&
|
|
152
|
+
"loadAndSplit" in arg &&
|
|
153
|
+
typeof arg.loadAndSplit === "function") {
|
|
154
|
+
return true;
|
|
155
|
+
}
|
|
156
|
+
return false;
|
|
157
|
+
};
|
|
158
|
+
exports._isBaseDocumentLoader = _isBaseDocumentLoader;
|
|
159
|
+
/**
|
|
160
|
+
* Index data from the doc source into the vector store.
|
|
161
|
+
*
|
|
162
|
+
* Indexing functionality uses a manager to keep track of which documents
|
|
163
|
+
* are in the vector store.
|
|
164
|
+
*
|
|
165
|
+
* This allows us to keep track of which documents were updated, and which
|
|
166
|
+
* documents were deleted, which documents should be skipped.
|
|
167
|
+
*
|
|
168
|
+
* For the time being, documents are indexed using their hashes, and users
|
|
169
|
+
* are not able to specify the uid of the document.
|
|
170
|
+
*
|
|
171
|
+
* @param {IndexArgs} args
|
|
172
|
+
* @param {BaseDocumentLoader | DocumentInterface[]} args.docsSource The source of documents to index. Can be a DocumentLoader or a list of Documents.
|
|
173
|
+
* @param {RecordManagerInterface} args.recordManager The record manager to use for keeping track of indexed documents.
|
|
174
|
+
* @param {VectorStore} args.vectorStore The vector store to use for storing the documents.
|
|
175
|
+
* @param {IndexOptions | undefined} args.options Options for indexing.
|
|
176
|
+
* @returns {Promise<IndexingResult>}
|
|
177
|
+
*/
|
|
178
|
+
async function index(args) {
|
|
179
|
+
const { docsSource, recordManager, vectorStore, options } = args;
|
|
180
|
+
const { batchSize = 100, cleanup, sourceIdKey, cleanupBatchSize = 1000, forceUpdate = false, } = options ?? {};
|
|
181
|
+
if (cleanup === "incremental" && !sourceIdKey) {
|
|
182
|
+
throw new Error("sourceIdKey is required when cleanup mode is incremental. Please provide through 'options.sourceIdKey'.");
|
|
183
|
+
}
|
|
184
|
+
const docs = (0, exports._isBaseDocumentLoader)(docsSource)
|
|
185
|
+
? await docsSource.load()
|
|
186
|
+
: docsSource;
|
|
187
|
+
const sourceIdAssigner = _getSourceIdAssigner(sourceIdKey ?? null);
|
|
188
|
+
const indexStartDt = await recordManager.getTime();
|
|
189
|
+
let numAdded = 0;
|
|
190
|
+
let numDeleted = 0;
|
|
191
|
+
let numUpdated = 0;
|
|
192
|
+
let numSkipped = 0;
|
|
193
|
+
const batches = _batch(batchSize ?? 100, docs);
|
|
194
|
+
for (const batch of batches) {
|
|
195
|
+
const hashedDocs = _deduplicateInOrder(batch.map((doc) => _HashedDocument.fromDocument(doc)));
|
|
196
|
+
const sourceIds = hashedDocs.map((doc) => sourceIdAssigner(doc));
|
|
197
|
+
if (cleanup === "incremental") {
|
|
198
|
+
hashedDocs.forEach((_hashedDoc, index) => {
|
|
199
|
+
const source = sourceIds[index];
|
|
200
|
+
if (source === null) {
|
|
201
|
+
throw new Error("sourceIdKey must be provided when cleanup is incremental");
|
|
202
|
+
}
|
|
203
|
+
});
|
|
204
|
+
}
|
|
205
|
+
const batchExists = await recordManager.exists(hashedDocs.map((doc) => doc.uid));
|
|
206
|
+
const uids = [];
|
|
207
|
+
const docsToIndex = [];
|
|
208
|
+
const docsToUpdate = [];
|
|
209
|
+
const seenDocs = new Set();
|
|
210
|
+
hashedDocs.forEach((hashedDoc, i) => {
|
|
211
|
+
const docExists = batchExists[i];
|
|
212
|
+
if (docExists) {
|
|
213
|
+
if (forceUpdate) {
|
|
214
|
+
seenDocs.add(hashedDoc.uid);
|
|
215
|
+
}
|
|
216
|
+
else {
|
|
217
|
+
docsToUpdate.push(hashedDoc.uid);
|
|
218
|
+
return;
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
uids.push(hashedDoc.uid);
|
|
222
|
+
docsToIndex.push(hashedDoc.toDocument());
|
|
223
|
+
});
|
|
224
|
+
if (docsToUpdate.length > 0) {
|
|
225
|
+
await recordManager.update(docsToUpdate, { timeAtLeast: indexStartDt });
|
|
226
|
+
numSkipped += docsToUpdate.length;
|
|
227
|
+
}
|
|
228
|
+
if (docsToIndex.length > 0) {
|
|
229
|
+
await vectorStore.addDocuments(docsToIndex, { ids: uids });
|
|
230
|
+
numAdded += docsToIndex.length - seenDocs.size;
|
|
231
|
+
numUpdated += seenDocs.size;
|
|
232
|
+
}
|
|
233
|
+
await recordManager.update(hashedDocs.map((doc) => doc.uid), { timeAtLeast: indexStartDt, groupIds: sourceIds });
|
|
234
|
+
if (cleanup === "incremental") {
|
|
235
|
+
sourceIds.forEach((sourceId) => {
|
|
236
|
+
if (!sourceId)
|
|
237
|
+
throw new Error("Source id cannot be null");
|
|
238
|
+
});
|
|
239
|
+
const uidsToDelete = await recordManager.listKeys({
|
|
240
|
+
before: indexStartDt,
|
|
241
|
+
groupIds: sourceIds,
|
|
242
|
+
});
|
|
243
|
+
await vectorStore.delete({ ids: uidsToDelete });
|
|
244
|
+
await recordManager.deleteKeys(uidsToDelete);
|
|
245
|
+
numDeleted += uidsToDelete.length;
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
if (cleanup === "full") {
|
|
249
|
+
let uidsToDelete = await recordManager.listKeys({
|
|
250
|
+
before: indexStartDt,
|
|
251
|
+
limit: cleanupBatchSize,
|
|
252
|
+
});
|
|
253
|
+
while (uidsToDelete.length > 0) {
|
|
254
|
+
await vectorStore.delete({ ids: uidsToDelete });
|
|
255
|
+
await recordManager.deleteKeys(uidsToDelete);
|
|
256
|
+
numDeleted += uidsToDelete.length;
|
|
257
|
+
uidsToDelete = await recordManager.listKeys({
|
|
258
|
+
before: indexStartDt,
|
|
259
|
+
limit: cleanupBatchSize,
|
|
260
|
+
});
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
return {
|
|
264
|
+
numAdded,
|
|
265
|
+
numDeleted,
|
|
266
|
+
numUpdated,
|
|
267
|
+
numSkipped,
|
|
268
|
+
};
|
|
269
|
+
}
|
|
270
|
+
exports.index = index;
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import { VectorStore } from "../vectorstores.js";
|
|
2
|
+
import { RecordManagerInterface } from "./record_manager.js";
|
|
3
|
+
import { DocumentInterface } from "../documents/document.js";
|
|
4
|
+
import { BaseDocumentLoader } from "../document_loaders/base.js";
|
|
5
|
+
type Metadata = Record<string, unknown>;
|
|
6
|
+
type IndexingResult = {
|
|
7
|
+
numAdded: number;
|
|
8
|
+
numDeleted: number;
|
|
9
|
+
numUpdated: number;
|
|
10
|
+
numSkipped: number;
|
|
11
|
+
};
|
|
12
|
+
type StringOrDocFunc = string | ((doc: DocumentInterface) => string);
|
|
13
|
+
export interface HashedDocumentInterface extends DocumentInterface {
|
|
14
|
+
uid: string;
|
|
15
|
+
hash_?: string;
|
|
16
|
+
contentHash?: string;
|
|
17
|
+
metadataHash?: string;
|
|
18
|
+
pageContent: string;
|
|
19
|
+
metadata: Metadata;
|
|
20
|
+
calculateHashes(): void;
|
|
21
|
+
toDocument(): DocumentInterface;
|
|
22
|
+
}
|
|
23
|
+
interface HashedDocumentArgs {
|
|
24
|
+
pageContent: string;
|
|
25
|
+
metadata: Metadata;
|
|
26
|
+
uid: string;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* HashedDocument is a Document with hashes calculated.
|
|
30
|
+
* Hashes are calculated based on page content and metadata.
|
|
31
|
+
* It is used for indexing.
|
|
32
|
+
*/
|
|
33
|
+
export declare class _HashedDocument implements HashedDocumentInterface {
|
|
34
|
+
uid: string;
|
|
35
|
+
hash_?: string;
|
|
36
|
+
contentHash?: string;
|
|
37
|
+
metadataHash?: string;
|
|
38
|
+
pageContent: string;
|
|
39
|
+
metadata: Metadata;
|
|
40
|
+
constructor(fields: HashedDocumentArgs);
|
|
41
|
+
calculateHashes(): void;
|
|
42
|
+
toDocument(): DocumentInterface;
|
|
43
|
+
static fromDocument(document: DocumentInterface, uid?: string): _HashedDocument;
|
|
44
|
+
private _hashStringToUUID;
|
|
45
|
+
private _hashNestedDictToUUID;
|
|
46
|
+
}
|
|
47
|
+
export type CleanupMode = "full" | "incremental";
|
|
48
|
+
export type IndexOptions = {
|
|
49
|
+
/**
|
|
50
|
+
* The number of documents to index in one batch.
|
|
51
|
+
*/
|
|
52
|
+
batchSize?: number;
|
|
53
|
+
/**
|
|
54
|
+
* The cleanup mode to use. Can be "full", "incremental" or undefined.
|
|
55
|
+
* - **Incremental**: Cleans up all documents that haven't been updated AND
|
|
56
|
+
* that are associated with source ids that were seen
|
|
57
|
+
* during indexing.
|
|
58
|
+
* Clean up is done continuously during indexing helping
|
|
59
|
+
* to minimize the probability of users seeing duplicated
|
|
60
|
+
* content.
|
|
61
|
+
* - **Full**: Delete all documents that haven to been returned by the loader.
|
|
62
|
+
* Clean up runs after all documents have been indexed.
|
|
63
|
+
* This means that users may see duplicated content during indexing.
|
|
64
|
+
* - **undefined**: Do not delete any documents.
|
|
65
|
+
*/
|
|
66
|
+
cleanup?: CleanupMode;
|
|
67
|
+
/**
|
|
68
|
+
* Optional key that helps identify the original source of the document.
|
|
69
|
+
* Must either be a string representing the key of the source in the metadata
|
|
70
|
+
* or a function that takes a document and returns a string representing the source.
|
|
71
|
+
* **Required when cleanup is incremental**.
|
|
72
|
+
*/
|
|
73
|
+
sourceIdKey?: StringOrDocFunc;
|
|
74
|
+
/**
|
|
75
|
+
* Batch size to use when cleaning up documents.
|
|
76
|
+
*/
|
|
77
|
+
cleanupBatchSize?: number;
|
|
78
|
+
/**
|
|
79
|
+
* Force update documents even if they are present in the
|
|
80
|
+
* record manager. Useful if you are re-indexing with updated embeddings.
|
|
81
|
+
*/
|
|
82
|
+
forceUpdate?: boolean;
|
|
83
|
+
};
|
|
84
|
+
export declare function _batch<T>(size: number, iterable: T[]): T[][];
|
|
85
|
+
export declare function _deduplicateInOrder(hashedDocuments: HashedDocumentInterface[]): HashedDocumentInterface[];
|
|
86
|
+
export declare function _getSourceIdAssigner(sourceIdKey: StringOrDocFunc | null): (doc: DocumentInterface) => string | null;
|
|
87
|
+
export declare const _isBaseDocumentLoader: (arg: any) => arg is BaseDocumentLoader;
|
|
88
|
+
interface IndexArgs {
|
|
89
|
+
docsSource: BaseDocumentLoader | DocumentInterface[];
|
|
90
|
+
recordManager: RecordManagerInterface;
|
|
91
|
+
vectorStore: VectorStore;
|
|
92
|
+
options?: IndexOptions;
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Index data from the doc source into the vector store.
|
|
96
|
+
*
|
|
97
|
+
* Indexing functionality uses a manager to keep track of which documents
|
|
98
|
+
* are in the vector store.
|
|
99
|
+
*
|
|
100
|
+
* This allows us to keep track of which documents were updated, and which
|
|
101
|
+
* documents were deleted, which documents should be skipped.
|
|
102
|
+
*
|
|
103
|
+
* For the time being, documents are indexed using their hashes, and users
|
|
104
|
+
* are not able to specify the uid of the document.
|
|
105
|
+
*
|
|
106
|
+
* @param {IndexArgs} args
|
|
107
|
+
* @param {BaseDocumentLoader | DocumentInterface[]} args.docsSource The source of documents to index. Can be a DocumentLoader or a list of Documents.
|
|
108
|
+
* @param {RecordManagerInterface} args.recordManager The record manager to use for keeping track of indexed documents.
|
|
109
|
+
* @param {VectorStore} args.vectorStore The vector store to use for storing the documents.
|
|
110
|
+
* @param {IndexOptions | undefined} args.options Options for indexing.
|
|
111
|
+
* @returns {Promise<IndexingResult>}
|
|
112
|
+
*/
|
|
113
|
+
export declare function index(args: IndexArgs): Promise<IndexingResult>;
|
|
114
|
+
export {};
|
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
import { v5 as uuidv5 } from "uuid";
|
|
2
|
+
import { UUIDV5_NAMESPACE } from "./record_manager.js";
|
|
3
|
+
import { insecureHash } from "../utils/hash.js";
|
|
4
|
+
import { Document } from "../documents/document.js";
|
|
5
|
+
/**
|
|
6
|
+
* HashedDocument is a Document with hashes calculated.
|
|
7
|
+
* Hashes are calculated based on page content and metadata.
|
|
8
|
+
* It is used for indexing.
|
|
9
|
+
*/
|
|
10
|
+
export class _HashedDocument {
|
|
11
|
+
constructor(fields) {
|
|
12
|
+
Object.defineProperty(this, "uid", {
|
|
13
|
+
enumerable: true,
|
|
14
|
+
configurable: true,
|
|
15
|
+
writable: true,
|
|
16
|
+
value: void 0
|
|
17
|
+
});
|
|
18
|
+
Object.defineProperty(this, "hash_", {
|
|
19
|
+
enumerable: true,
|
|
20
|
+
configurable: true,
|
|
21
|
+
writable: true,
|
|
22
|
+
value: void 0
|
|
23
|
+
});
|
|
24
|
+
Object.defineProperty(this, "contentHash", {
|
|
25
|
+
enumerable: true,
|
|
26
|
+
configurable: true,
|
|
27
|
+
writable: true,
|
|
28
|
+
value: void 0
|
|
29
|
+
});
|
|
30
|
+
Object.defineProperty(this, "metadataHash", {
|
|
31
|
+
enumerable: true,
|
|
32
|
+
configurable: true,
|
|
33
|
+
writable: true,
|
|
34
|
+
value: void 0
|
|
35
|
+
});
|
|
36
|
+
Object.defineProperty(this, "pageContent", {
|
|
37
|
+
enumerable: true,
|
|
38
|
+
configurable: true,
|
|
39
|
+
writable: true,
|
|
40
|
+
value: void 0
|
|
41
|
+
});
|
|
42
|
+
Object.defineProperty(this, "metadata", {
|
|
43
|
+
enumerable: true,
|
|
44
|
+
configurable: true,
|
|
45
|
+
writable: true,
|
|
46
|
+
value: void 0
|
|
47
|
+
});
|
|
48
|
+
this.uid = fields.uid;
|
|
49
|
+
this.pageContent = fields.pageContent;
|
|
50
|
+
this.metadata = fields.metadata;
|
|
51
|
+
}
|
|
52
|
+
calculateHashes() {
|
|
53
|
+
const forbiddenKeys = ["hash_", "content_hash", "metadata_hash"];
|
|
54
|
+
for (const key of forbiddenKeys) {
|
|
55
|
+
if (key in this.metadata) {
|
|
56
|
+
throw new Error(`Metadata cannot contain key ${key} as it is reserved for internal use. Restricted keys: [${forbiddenKeys.join(", ")}]`);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
const contentHash = this._hashStringToUUID(this.pageContent);
|
|
60
|
+
try {
|
|
61
|
+
const metadataHash = this._hashNestedDictToUUID(this.metadata);
|
|
62
|
+
this.contentHash = contentHash;
|
|
63
|
+
this.metadataHash = metadataHash;
|
|
64
|
+
}
|
|
65
|
+
catch (e) {
|
|
66
|
+
throw new Error(`Failed to hash metadata: ${e}. Please use a dict that can be serialized using json.`);
|
|
67
|
+
}
|
|
68
|
+
this.hash_ = this._hashStringToUUID(this.contentHash + this.metadataHash);
|
|
69
|
+
if (!this.uid) {
|
|
70
|
+
this.uid = this.hash_;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
toDocument() {
|
|
74
|
+
return new Document({
|
|
75
|
+
pageContent: this.pageContent,
|
|
76
|
+
metadata: this.metadata,
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
static fromDocument(document, uid) {
|
|
80
|
+
const doc = new this({
|
|
81
|
+
pageContent: document.pageContent,
|
|
82
|
+
metadata: document.metadata,
|
|
83
|
+
uid: uid || document.uid,
|
|
84
|
+
});
|
|
85
|
+
doc.calculateHashes();
|
|
86
|
+
return doc;
|
|
87
|
+
}
|
|
88
|
+
_hashStringToUUID(inputString) {
|
|
89
|
+
const hash_value = insecureHash(inputString);
|
|
90
|
+
return uuidv5(hash_value, UUIDV5_NAMESPACE);
|
|
91
|
+
}
|
|
92
|
+
_hashNestedDictToUUID(data) {
|
|
93
|
+
const serialized_data = JSON.stringify(data, Object.keys(data).sort());
|
|
94
|
+
const hash_value = insecureHash(serialized_data);
|
|
95
|
+
return uuidv5(hash_value, UUIDV5_NAMESPACE);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
export function _batch(size, iterable) {
|
|
99
|
+
const batches = [];
|
|
100
|
+
let currentBatch = [];
|
|
101
|
+
iterable.forEach((item) => {
|
|
102
|
+
currentBatch.push(item);
|
|
103
|
+
if (currentBatch.length >= size) {
|
|
104
|
+
batches.push(currentBatch);
|
|
105
|
+
currentBatch = [];
|
|
106
|
+
}
|
|
107
|
+
});
|
|
108
|
+
if (currentBatch.length > 0) {
|
|
109
|
+
batches.push(currentBatch);
|
|
110
|
+
}
|
|
111
|
+
return batches;
|
|
112
|
+
}
|
|
113
|
+
export function _deduplicateInOrder(hashedDocuments) {
|
|
114
|
+
const seen = new Set();
|
|
115
|
+
const deduplicated = [];
|
|
116
|
+
for (const hashedDoc of hashedDocuments) {
|
|
117
|
+
if (!hashedDoc.hash_) {
|
|
118
|
+
throw new Error("Hashed document does not have a hash");
|
|
119
|
+
}
|
|
120
|
+
if (!seen.has(hashedDoc.hash_)) {
|
|
121
|
+
seen.add(hashedDoc.hash_);
|
|
122
|
+
deduplicated.push(hashedDoc);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
return deduplicated;
|
|
126
|
+
}
|
|
127
|
+
export function _getSourceIdAssigner(sourceIdKey) {
|
|
128
|
+
if (sourceIdKey === null) {
|
|
129
|
+
return (_doc) => null;
|
|
130
|
+
}
|
|
131
|
+
else if (typeof sourceIdKey === "string") {
|
|
132
|
+
return (doc) => doc.metadata[sourceIdKey];
|
|
133
|
+
}
|
|
134
|
+
else if (typeof sourceIdKey === "function") {
|
|
135
|
+
return sourceIdKey;
|
|
136
|
+
}
|
|
137
|
+
else {
|
|
138
|
+
throw new Error(`sourceIdKey should be null, a string or a function, got ${typeof sourceIdKey}`);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
142
|
+
export const _isBaseDocumentLoader = (arg) => {
|
|
143
|
+
if ("load" in arg &&
|
|
144
|
+
typeof arg.load === "function" &&
|
|
145
|
+
"loadAndSplit" in arg &&
|
|
146
|
+
typeof arg.loadAndSplit === "function") {
|
|
147
|
+
return true;
|
|
148
|
+
}
|
|
149
|
+
return false;
|
|
150
|
+
};
|
|
151
|
+
/**
|
|
152
|
+
* Index data from the doc source into the vector store.
|
|
153
|
+
*
|
|
154
|
+
* Indexing functionality uses a manager to keep track of which documents
|
|
155
|
+
* are in the vector store.
|
|
156
|
+
*
|
|
157
|
+
* This allows us to keep track of which documents were updated, and which
|
|
158
|
+
* documents were deleted, which documents should be skipped.
|
|
159
|
+
*
|
|
160
|
+
* For the time being, documents are indexed using their hashes, and users
|
|
161
|
+
* are not able to specify the uid of the document.
|
|
162
|
+
*
|
|
163
|
+
* @param {IndexArgs} args
|
|
164
|
+
* @param {BaseDocumentLoader | DocumentInterface[]} args.docsSource The source of documents to index. Can be a DocumentLoader or a list of Documents.
|
|
165
|
+
* @param {RecordManagerInterface} args.recordManager The record manager to use for keeping track of indexed documents.
|
|
166
|
+
* @param {VectorStore} args.vectorStore The vector store to use for storing the documents.
|
|
167
|
+
* @param {IndexOptions | undefined} args.options Options for indexing.
|
|
168
|
+
* @returns {Promise<IndexingResult>}
|
|
169
|
+
*/
|
|
170
|
+
export async function index(args) {
|
|
171
|
+
const { docsSource, recordManager, vectorStore, options } = args;
|
|
172
|
+
const { batchSize = 100, cleanup, sourceIdKey, cleanupBatchSize = 1000, forceUpdate = false, } = options ?? {};
|
|
173
|
+
if (cleanup === "incremental" && !sourceIdKey) {
|
|
174
|
+
throw new Error("sourceIdKey is required when cleanup mode is incremental. Please provide through 'options.sourceIdKey'.");
|
|
175
|
+
}
|
|
176
|
+
const docs = _isBaseDocumentLoader(docsSource)
|
|
177
|
+
? await docsSource.load()
|
|
178
|
+
: docsSource;
|
|
179
|
+
const sourceIdAssigner = _getSourceIdAssigner(sourceIdKey ?? null);
|
|
180
|
+
const indexStartDt = await recordManager.getTime();
|
|
181
|
+
let numAdded = 0;
|
|
182
|
+
let numDeleted = 0;
|
|
183
|
+
let numUpdated = 0;
|
|
184
|
+
let numSkipped = 0;
|
|
185
|
+
const batches = _batch(batchSize ?? 100, docs);
|
|
186
|
+
for (const batch of batches) {
|
|
187
|
+
const hashedDocs = _deduplicateInOrder(batch.map((doc) => _HashedDocument.fromDocument(doc)));
|
|
188
|
+
const sourceIds = hashedDocs.map((doc) => sourceIdAssigner(doc));
|
|
189
|
+
if (cleanup === "incremental") {
|
|
190
|
+
hashedDocs.forEach((_hashedDoc, index) => {
|
|
191
|
+
const source = sourceIds[index];
|
|
192
|
+
if (source === null) {
|
|
193
|
+
throw new Error("sourceIdKey must be provided when cleanup is incremental");
|
|
194
|
+
}
|
|
195
|
+
});
|
|
196
|
+
}
|
|
197
|
+
const batchExists = await recordManager.exists(hashedDocs.map((doc) => doc.uid));
|
|
198
|
+
const uids = [];
|
|
199
|
+
const docsToIndex = [];
|
|
200
|
+
const docsToUpdate = [];
|
|
201
|
+
const seenDocs = new Set();
|
|
202
|
+
hashedDocs.forEach((hashedDoc, i) => {
|
|
203
|
+
const docExists = batchExists[i];
|
|
204
|
+
if (docExists) {
|
|
205
|
+
if (forceUpdate) {
|
|
206
|
+
seenDocs.add(hashedDoc.uid);
|
|
207
|
+
}
|
|
208
|
+
else {
|
|
209
|
+
docsToUpdate.push(hashedDoc.uid);
|
|
210
|
+
return;
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
uids.push(hashedDoc.uid);
|
|
214
|
+
docsToIndex.push(hashedDoc.toDocument());
|
|
215
|
+
});
|
|
216
|
+
if (docsToUpdate.length > 0) {
|
|
217
|
+
await recordManager.update(docsToUpdate, { timeAtLeast: indexStartDt });
|
|
218
|
+
numSkipped += docsToUpdate.length;
|
|
219
|
+
}
|
|
220
|
+
if (docsToIndex.length > 0) {
|
|
221
|
+
await vectorStore.addDocuments(docsToIndex, { ids: uids });
|
|
222
|
+
numAdded += docsToIndex.length - seenDocs.size;
|
|
223
|
+
numUpdated += seenDocs.size;
|
|
224
|
+
}
|
|
225
|
+
await recordManager.update(hashedDocs.map((doc) => doc.uid), { timeAtLeast: indexStartDt, groupIds: sourceIds });
|
|
226
|
+
if (cleanup === "incremental") {
|
|
227
|
+
sourceIds.forEach((sourceId) => {
|
|
228
|
+
if (!sourceId)
|
|
229
|
+
throw new Error("Source id cannot be null");
|
|
230
|
+
});
|
|
231
|
+
const uidsToDelete = await recordManager.listKeys({
|
|
232
|
+
before: indexStartDt,
|
|
233
|
+
groupIds: sourceIds,
|
|
234
|
+
});
|
|
235
|
+
await vectorStore.delete({ ids: uidsToDelete });
|
|
236
|
+
await recordManager.deleteKeys(uidsToDelete);
|
|
237
|
+
numDeleted += uidsToDelete.length;
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
if (cleanup === "full") {
|
|
241
|
+
let uidsToDelete = await recordManager.listKeys({
|
|
242
|
+
before: indexStartDt,
|
|
243
|
+
limit: cleanupBatchSize,
|
|
244
|
+
});
|
|
245
|
+
while (uidsToDelete.length > 0) {
|
|
246
|
+
await vectorStore.delete({ ids: uidsToDelete });
|
|
247
|
+
await recordManager.deleteKeys(uidsToDelete);
|
|
248
|
+
numDeleted += uidsToDelete.length;
|
|
249
|
+
uidsToDelete = await recordManager.listKeys({
|
|
250
|
+
before: indexStartDt,
|
|
251
|
+
limit: cleanupBatchSize,
|
|
252
|
+
});
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
return {
|
|
256
|
+
numAdded,
|
|
257
|
+
numDeleted,
|
|
258
|
+
numUpdated,
|
|
259
|
+
numSkipped,
|
|
260
|
+
};
|
|
261
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
__exportStar(require("./record_manager.cjs"), exports);
|
|
18
|
+
__exportStar(require("./base.cjs"), exports);
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.RecordManager = exports.UUIDV5_NAMESPACE = void 0;
|
|
4
|
+
const serializable_js_1 = require("../load/serializable.cjs");
|
|
5
|
+
// Arbitrary value, used for generating namespaced UUIDs.
|
|
6
|
+
exports.UUIDV5_NAMESPACE = "10f90ea3-90a4-4962-bf75-83a0f3c1c62a";
|
|
7
|
+
class RecordManager extends serializable_js_1.Serializable {
|
|
8
|
+
constructor() {
|
|
9
|
+
super(...arguments);
|
|
10
|
+
Object.defineProperty(this, "lc_namespace", {
|
|
11
|
+
enumerable: true,
|
|
12
|
+
configurable: true,
|
|
13
|
+
writable: true,
|
|
14
|
+
value: ["langchain", "recordmanagers"]
|
|
15
|
+
});
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
exports.RecordManager = RecordManager;
|