datastore-api 6.2.0 → 6.2.2

package/src/lib/dstore-api-cloud.spec.ts

@@ -5,26 +5,26 @@
  * Copyright (c) 2021, 2023 Dr. Maximilian Dornseif
  */
 // @ts-nocheck
-import { Datastore, Key } from '@google-cloud/datastore';
-import { assert, describe, expect, test } from 'vitest';
+import { Datastore, Key } from '@google-cloud/datastore'
+import { assert, describe, expect, test } from 'vitest'
 
-import { Dstore } from './dstore-api';
+import { Dstore } from './dstore-api'
 
 function getDstore() {
-  return new Dstore(new Datastore({ namespace: 'test', projectId: 'hdmashup-hrd' }));
+  return new Dstore(new Datastore({ namespace: 'test', projectId: 'hdmashup-hrd' }))
 }
 
 test('keySerialize', async () => {
-  const kvStore = getDstore();
-  assert.deepEqual(kvStore.key(['testYodel', 123]).path, ['testYodel', 123]);
+  const kvStore = getDstore()
+  assert.deepEqual(kvStore.key(['testYodel', 123]).path, ['testYodel', 123])
   assert.deepEqual(JSON.parse(JSON.stringify(kvStore.key(['testYodel', 123]))), {
     id: 123 as any, // typing in inconclusive here
     kind: 'testYodel',
     namespace: 'test',
     path: ['testYodel', 123],
-  } as any);
-  const ser = kvStore.keySerialize(kvStore.key(['testYodel', 123]));
-  expect(ser).toMatchInlineSnapshot('"agByDwsSCXRlc3RZb2RlbBh7DKIBBHRlc3Q"');
+  } as any)
+  const ser = kvStore.keySerialize(kvStore.key(['testYodel', 123]))
+  expect(ser).toMatchInlineSnapshot('"agByDwsSCXRlc3RZb2RlbBh7DKIBBHRlc3Q"')
   expect(JSON.parse(JSON.stringify(kvStore.keyFromSerialized(ser)))).toMatchInlineSnapshot(`
     {
       "id": "123",
@@ -35,39 +35,39 @@ test('keySerialize', async () => {
         "123",
       ],
     }
-  `);
-});
+  `)
+})
 
 describe('Allocation', () => {
   test('allocateIds', async () => {
-    const kvStore = getDstore();
-    const keys = await kvStore.datastore.allocateIds(kvStore.datastore.key(['testYodel']), 2);
-    expect(Array.isArray(keys)).toBeTruthy();
-    expect(keys[0].length).toBe(2);
-    expect(keys[0][0].kind).toBe('testYodel');
-    expect(keys[0][0].id).toMatch(/\d+/);
-    expect(keys?.[1]?.keys?.length).toBe(2);
-    expect(keys[1].keys[0].partitionId.namespaceId).toMatchInlineSnapshot('"test"');
-    expect(keys[1].keys[0].path[0].idType).toMatchInlineSnapshot('"id"');
-    expect(keys[1].keys[0].path[0].kind).toMatchInlineSnapshot('"testYodel"');
-  });
+    const kvStore = getDstore()
+    const keys = await kvStore.datastore.allocateIds(kvStore.datastore.key(['testYodel']), 2)
+    expect(Array.isArray(keys)).toBeTruthy()
+    expect(keys[0].length).toBe(2)
+    expect(keys[0][0].kind).toBe('testYodel')
+    expect(keys[0][0].id).toMatch(/\d+/)
+    expect(keys?.[1]?.keys?.length).toBe(2)
+    expect(keys[1].keys[0].partitionId.namespaceId).toMatchInlineSnapshot('"test"')
+    expect(keys[1].keys[0].path[0].idType).toMatchInlineSnapshot('"id"')
+    expect(keys[1].keys[0].path[0].kind).toMatchInlineSnapshot('"testYodel"')
+  })
 
   test('allocateOneId', async () => {
-    const kvStore = getDstore();
-    const id = await kvStore.allocateOneId();
-    expect(id).toMatch(/\d+/);
-  });
-});
+    const kvStore = getDstore()
+    const id = await kvStore.allocateOneId()
+    expect(id).toMatch(/\d+/)
+  })
+})
 
 describe('Read', () => {
   test('get num_id', async () => {
-    const kvStore = getDstore();
-    const entity = { key: kvStore.key(['testYodel', 2]), data: { foo: 'bar' } };
+    const kvStore = getDstore()
+    const entity = { key: kvStore.key(['testYodel', 2]), data: { foo: 'bar' } }
     const entity2 = {
       key: kvStore.key(['testYodel', 3]),
       data: { foo: 'bar' },
-    };
-    const commitResponse = await kvStore.save([entity, entity2]);
+    }
+    const commitResponse = await kvStore.save([entity, entity2])
     // expect(isNumber(commitResponse?.[0]?.indexUpdates)).toBeTruthy();
     // expect(commitResponse).toMatchInlineSnapshot(`
     //   Array [
@@ -109,22 +109,22 @@ describe('Read', () => {
           ],
         },
       }
-    `);
+    `)
 
-    const result = await kvStore.get(entity.key);
+    const result = await kvStore.get(entity.key)
     // get returns a single Entity
-    expect(Array.isArray(result)).toBeFalsy();
-    expect(result).toMatchInlineSnapshot('null');
+    expect(Array.isArray(result)).toBeFalsy()
+    expect(result).toMatchInlineSnapshot('null')
     // expect(kvStore.readKey(result)).toBeInstanceOf(Key);
 
-    const result2 = await kvStore.getMulti([entity.key]);
+    const result2 = await kvStore.getMulti([entity.key])
     // getMulti returns a Array even for single keys
     expect(result2).toMatchInlineSnapshot(`
       [
         null,
       ]
-    `);
-    const result3 = await kvStore.getMulti([entity.key, kvStore.key(['testYodel', 3])]);
+    `)
+    const result3 = await kvStore.getMulti([entity.key, kvStore.key(['testYodel', 3])])
     // getMulti returns a Array with multiple keys
     // expect(Array.isArray(result)).toBeTruthy();
     expect(result3).toMatchInlineSnapshot(`
@@ -132,8 +132,8 @@ describe('Read', () => {
         null,
         null,
       ]
-    `);
-    const result4 = await kvStore.getMulti([entity.key, entity.key]);
+    `)
+    const result4 = await kvStore.getMulti([entity.key, entity.key])
     // getMulti returns a Array but collapses duplicate keys
     // expect(Array.isArray(result)).toBeTruthy();
     // Firestore in Datastore returns the entity once
@@ -144,9 +144,9 @@ describe('Read', () => {
         null,
         null,
       ]
-    `);
+    `)
 
-    const result5 = await kvStore.getMulti([entity.key, kvStore.key(['YodelNotThere', 3])]);
+    const result5 = await kvStore.getMulti([entity.key, kvStore.key(['YodelNotThere', 3])])
     // getMulti returns a Array but omits unknown keys
     // expect(Array.isArray(result)).toBeTruthy();
     expect(result5).toMatchInlineSnapshot(`
@@ -154,39 +154,39 @@ describe('Read', () => {
         null,
         null,
       ]
-    `);
-    const result6 = await kvStore.getMulti([]);
+    `)
+    const result6 = await kvStore.getMulti([])
     // getMulti returns a empty Array for an empty array
     // expect(Array.isArray(result)).toBeTruthy();
-    expect(result6).toMatchInlineSnapshot('[]');
-  });
-});
+    expect(result6).toMatchInlineSnapshot('[]')
+  })
+})
 
 test('get name', async (t) => {
-  const kvStore = getDstore();
+  const kvStore = getDstore()
   const entity = {
     key: kvStore.key(['testYodel', 'two']),
     data: { foo: 'bar' },
-  };
-  await kvStore.save([entity]);
-  const result = await kvStore.get(entity.key);
-  expect(result?._keyStr).toMatchInlineSnapshot('"agByEgsSCXRlc3RZb2RlbCIDdHdvDKIBBHRlc3Q"');
-  expect(result?.foo).toBe('bar');
-});
+  }
+  await kvStore.save([entity])
+  const result = await kvStore.get(entity.key)
+  expect(result?._keyStr).toMatchInlineSnapshot('"agByEgsSCXRlc3RZb2RlbCIDdHdvDKIBBHRlc3Q"')
+  expect(result?.foo).toBe('bar')
+})
 
 describe('query', async () => {
   test('raw', async () => {
-    const kvStore = getDstore();
+    const kvStore = getDstore()
     const entity = {
       key: kvStore.key(['testYodel', '3']),
       data: { foo: 'bar', baz: 'baz' },
-    };
+    }
 
-    await kvStore.save([entity]);
-    const query = kvStore.datastore.createQuery('testYodel');
-    query.limit(1);
-    const [entities, runQueryInfo] = await kvStore.datastore.runQuery(query);
-    expect(entities.length).toBe(1);
+    await kvStore.save([entity])
+    const query = kvStore.datastore.createQuery('testYodel')
+    query.limit(1)
+    const [entities, runQueryInfo] = await kvStore.datastore.runQuery(query)
+    expect(entities.length).toBe(1)
     expect(entities).toMatchInlineSnapshot(`
       [
         {
@@ -202,18 +202,18 @@ describe('query', async () => {
           },
         },
       ]
-    `);
-  });
+    `)
+  })
 
   test('query', async () => {
-    const kvStore = getDstore();
+    const kvStore = getDstore()
     const entity = {
       key: kvStore.key(['testYodel', '3']),
       data: { foo: 'bar', baz: 'baz' },
-    };
+    }
 
-    const saveResult = await kvStore.save([entity]);
-    expect(saveResult?.[0]?.mutationResults?.key).toMatchInlineSnapshot('undefined');
+    const saveResult = await kvStore.save([entity])
+    expect(saveResult?.[0]?.mutationResults?.key).toMatchInlineSnapshot('undefined')
 
     // expect(saveResult).toMatchInlineSnapshot(`
     //   [
@@ -253,13 +253,13 @@ describe('query', async () => {
           ],
         },
       }
-    `);
+    `)
     // Give Datastore time to become consistent
-    do {} while ((await kvStore.get(entity.key)) === null);
+    do { } while ((await kvStore.get(entity.key)) === null)
 
-    const query = kvStore.createQuery('testYodel');
-    query.limit(1);
-    const [entities, runQueryInfo] = await kvStore.runQuery(query);
+    const query = kvStore.createQuery('testYodel')
+    query.limit(1)
+    const [entities, runQueryInfo] = await kvStore.runQuery(query)
     // expect(entities.length).toBe(1)
     expect(entities).toMatchInlineSnapshot(`
       [
@@ -277,112 +277,112 @@ describe('query', async () => {
           },
         },
       ]
-    `);
+    `)
     expect(runQueryInfo).toMatchInlineSnapshot(`
       {
         "endCursor": "Ci4SKGoOc35oZG1hc2h1cC1ocmRyDwsSCXRlc3RZb2RlbBgCDKIBBHRlc3QYACAA",
         "moreResults": "MORE_RESULTS_AFTER_LIMIT",
       }
-    `);
-    expect(entities?.[0]?.foo).toBe('bar');
-    expect(entities?.[0]?.[Datastore.KEY]?.kind).toBe('testYodel');
-    expect(runQueryInfo?.moreResults).toBe('MORE_RESULTS_AFTER_LIMIT');
+    `)
+    expect(entities?.[0]?.foo).toBe('bar')
+    expect(entities?.[0]?.[Datastore.KEY]?.kind).toBe('testYodel')
+    expect(runQueryInfo?.moreResults).toBe('MORE_RESULTS_AFTER_LIMIT')
 
     // modern interface
-    const [result2] = await kvStore.query('testYodel', [], 1, [], ['baz']);
-    expect(result2.length).toBe(1);
+    const [result2] = await kvStore.query('testYodel', [], 1, [], ['baz'])
+    expect(result2.length).toBe(1)
     // foo is removed by selection
     expect(JSON.parse(JSON.stringify(result2?.[0]))).toMatchInlineSnapshot(`
       {
         "_keyStr": "agByEAsSCXRlc3RZb2RlbCIBMwyiAQR0ZXN0",
         "baz": "baz",
       }
-    `);
+    `)
 
-    const key = kvStore.readKey(result2?.[0]);
-    expect(key.id).toBe(entity.key.id);
-  });
-});
+    const key = kvStore.readKey(result2?.[0])
+    expect(key.id).toBe(entity.key.id)
+  })
+})
 
 test('set', async () => {
   // expect.assertions(2);
-  const kvStore = getDstore();
+  const kvStore = getDstore()
   const result = await kvStore.set(kvStore.key(['testYodel', '5e7']), {
     foo: 'bar',
-  });
-  expect(result.name).toBe('5e7');
-  expect(result.kind).toBe('testYodel');
+  })
+  expect(result.name).toBe('5e7')
+  expect(result.kind).toBe('testYodel')
 
   // autogenerate key
-  const result2 = await kvStore.set(kvStore.key(['testYodel']), { foo: 'bar' });
-  expect(result2.kind).toBe('testYodel');
-});
+  const result2 = await kvStore.set(kvStore.key(['testYodel']), { foo: 'bar' })
+  expect(result2.kind).toBe('testYodel')
+})
 
 test('save / upsert', async () => {
   // expect.assertions(2);
-  const kvStore = getDstore();
+  const kvStore = getDstore()
   const entity = {
     key: kvStore.key(['testYodel', 3]),
     data: { foo: 'bar' } as any,
-  };
-  const result = await kvStore.save([entity]);
+  }
+  const result = await kvStore.save([entity])
   // const result2 = await kvStore.upsert([entity]);
-  expect(result?.[0]?.mutationResults?.[0]?.conflictDetected).toBe(false);
-  expect(entity.data._keyStr).toMatchInlineSnapshot('"agByDwsSCXRlc3RZb2RlbBgDDKIBBHRlc3Q"');
-  expect(entity.data.foo).toBe('bar');
-  expect(entity.data[Datastore.KEY].kind).toBe('testYodel');
-});
+  expect(result?.[0]?.mutationResults?.[0]?.conflictDetected).toBe(false)
+  expect(entity.data._keyStr).toMatchInlineSnapshot('"agByDwsSCXRlc3RZb2RlbBgDDKIBBHRlc3Q"')
+  expect(entity.data.foo).toBe('bar')
+  expect(entity.data[Datastore.KEY].kind).toBe('testYodel')
+})
 
 test('update', async (t) => {
   //     expect.assertions(3);
-  const kvStore = getDstore();
-  const keyName = `4insert${Math.random()}`;
+  const kvStore = getDstore()
+  const keyName = `4insert${Math.random()}`
   const entity = {
     key: kvStore.key(['testYodel', keyName]),
     data: { foo: 'bar' },
-  };
+  }
   // const request = kvStore.update([entity]);
   // await expect(request).rejects.toThrowError(Error);
 
-  await kvStore.save([entity]);
-  const result = await kvStore.update([entity]);
-  expect(result?.[0]?.mutationResults?.[0]?.conflictDetected).toBe(false);
-  expect(result?.[0]?.mutationResults?.[0]?.key).toBe(null);
+  await kvStore.save([entity])
+  const result = await kvStore.update([entity])
+  expect(result?.[0]?.mutationResults?.[0]?.conflictDetected).toBe(false)
+  expect(result?.[0]?.mutationResults?.[0]?.key).toBe(null)
   // expect(result?.[0]?.indexUpdates).toBe(2);
-});
+})
 
 test('insert / delete', async (t) => {
   // expect.assertions(2);
-  const kvStore = getDstore();
-  const testkey = kvStore.key(['testYodel', 4]);
-  await kvStore.delete([testkey]);
+  const kvStore = getDstore()
+  const testkey = kvStore.key(['testYodel', 4])
+  await kvStore.delete([testkey])
   const entity = {
     key: testkey,
     data: { foo: 'bar' } as any,
-  };
-  const result = await kvStore.insert([entity]);
+  }
+  const result = await kvStore.insert([entity])
 
-  expect(result?.[0]?.mutationResults?.[0]?.conflictDetected).toBe(false);
-  expect(result?.[0]?.mutationResults?.[0]?.version).toMatch(/\d+/);
-  expect(entity.data.foo).toBe('bar');
-  expect(entity.key.path[0]).toBe('testYodel');
+  expect(result?.[0]?.mutationResults?.[0]?.conflictDetected).toBe(false)
+  expect(result?.[0]?.mutationResults?.[0]?.version).toMatch(/\d+/)
+  expect(entity.data.foo).toBe('bar')
+  expect(entity.key.path[0]).toBe('testYodel')
   // expect(result?.[0]?.indexUpdates).toBe(3);
 
-  const result2 = await kvStore.delete([entity.key]);
-  expect(result2?.[0]?.mutationResults?.[0]?.conflictDetected).toBe(false);
-});
+  const result2 = await kvStore.delete([entity.key])
+  expect(result2?.[0]?.mutationResults?.[0]?.conflictDetected).toBe(false)
+})
 
 test('exception', async () => {
   // expect.assertions(2);
-  const kvStore = getDstore();
+  const kvStore = getDstore()
   try {
     const result = await kvStore.set(kvStore.key(['testYodel', NaN]), {
       foo: 'bar',
-    });
+    })
   } catch (e) {
-    expect(e.stack).toMatch(/Dstore\.set/);
+    expect(e.stack).toMatch(/Dstore\.set/)
   }
-});
+})
 // describe("Transactions", () => {
 //   it("simple", async () => {
 //     expect.assertions(2);

package/src/lib/dstore-api.ts

@@ -28,6 +28,7 @@ import {
 import Debug from 'debug'
 import promClient from 'prom-client'
 import { Writable } from 'ts-essentials'
+import { assertIsKey } from './assert'
 
 /** @ignore */
 export { Datastore, Key, PathType, Query, Transaction } from '@google-cloud/datastore'
@@ -94,6 +95,15 @@ export interface IDstoreEntry extends IDstoreEntryWithoutKey {
   _keyStr: string
 }
 
+/** Represents what is actually stored inside the Datastore, called "Entity" by Google
+*/
+export interface IDstoreEntryWithKey extends IDstoreEntry {
+  /* Datastore Key provided by [@google-cloud/datastore](https://github.com/googleapis/nodejs-datastore#readme) */
+  readonly [Datastore.KEY]: Key
+  /** [Datastore.KEY] key */
+  _keyStr: string
+}
+
 /** Represents the thing you pass to the save method. Also called "Entity" by Google */
 export type DstoreSaveEntity = {
   key: Key
@@ -138,7 +148,7 @@ type IDstore = {
     selection?: readonly string[],
     cursor?: string
   ) => Promise<RunQueryResponse>
-  iterate: (options: IIterateParams) => AsyncIterable<IDstoreEntry>
+  iterate: (options: IIterateParams) => AsyncIterable<IDstoreEntryWithKey>
   allocateOneId: (kindName: string) => Promise<string>
   runInTransaction: <T>(func: { (): Promise<T>; (): T }) => Promise<T>
 }
@@ -271,6 +281,29 @@ export class Dstore implements IDstore {
     return entities as Array<IDstoreEntry | undefined>
   }
 
+  /** this is for save, insert, update and upsert  and ensures _kkeyStr() handling.
+   * 
+   */
+  private prepareEntitiesForDatastore(entities: readonly DstoreSaveEntity[]) {
+    for (const e of entities) {
+      assertIsObject(e.key)
+      assertIsObject(e.data)
+      this.fixKeys([e.data])
+      e.excludeLargeProperties = e.excludeLargeProperties === undefined ? true : e.excludeLargeProperties
+      e.data = { ...e.data, _keyStr: undefined }
+    }
+  }
+
+  /** this is for save, insert, update and upsert  and ensures _kkeyStr() handling.
+   * 
+   */
+  private prepareEntitiesFromDatastore(entities: readonly DstoreSaveEntity[] & unknown[]) {
+    for (const e of entities) {
+      e.data[Datastore.KEY] = e.key
+      this.fixKeys([e.data])
+    }
+  }
+
   /** `get()` reads a [[IDstoreEntry]] from the Datastore.
    *
    * It returns [[IDstoreEntry]] or `null` if not found.
@@ -399,20 +432,11 @@ export class Dstore implements IDstore {
     let ret: CommitResponse | undefined
     const metricEnd = metricHistogram.startTimer()
     try {
+      this.prepareEntitiesForDatastore(entities)
       // Within Transaction we don't get any answer here!
       // [ { mutationResults: [ [Object], [Object] ], indexUpdates: 51 } ]
-      for (const e of entities) {
-        assertIsObject(e.key)
-        assertIsObject(e.data)
-        this.fixKeys([e.data])
-        e.excludeLargeProperties = e.excludeLargeProperties === undefined ? true : e.excludeLargeProperties
-        e.data = { ...e.data, _keyStr: undefined }
-      }
       ret = (await this.getDoT().save(entities)) || undefined
-      for (const e of entities) {
-        e.data[Datastore.KEY] = e.key
-        this.fixKeys([e.data])
-      }
+      this.prepareEntitiesFromDatastore(entities)
     } catch (error) {
       metricFailureCounter.inc({ operation: 'save' })
       await setImmediate()
@@ -423,6 +447,8 @@ export class Dstore implements IDstore {
     return ret
   }
 
+
+
   /** `insert()` is compatible to [Datastore.insert()](https://cloud.google.com/nodejs/docs/reference/datastore/latest/datastore/datastore#_google_cloud_datastore_Datastore_insert_member_1_).
    *
    * The single Parameter is a list of [[DstoreSaveEntity]]s.
@@ -446,7 +472,9 @@ export class Dstore implements IDstore {
     let ret: CommitResponse | undefined
     const metricEnd = metricHistogram.startTimer()
     try {
+      this.prepareEntitiesForDatastore(entities)
       ret = (await this.getDoT().insert(entities)) || undefined
+      this.prepareEntitiesFromDatastore(entities)
     } catch (error) {
       // console.error(error)
       metricFailureCounter.inc({ operation: 'insert' })
@@ -490,7 +518,9 @@ export class Dstore implements IDstore {
     const metricEnd = metricHistogram.startTimer()
 
     try {
+      this.prepareEntitiesForDatastore(entities)
       ret = (await this.getDoT().update(entities)) || undefined
+      this.prepareEntitiesFromDatastore(entities)
     } catch (error) {
       // console.error(error)
       metricFailureCounter.inc({ operation: 'update' })
@@ -606,7 +636,8 @@ export class Dstore implements IDstore {
       if (selection.length > 0) {
         q.select(selection as any)
       }
-      return await this.runQuery(q)
+      const ret = await this.getDoT().runQuery(q)
+      return [this.fixKeys(ret[0]), ret[1]]
     } catch (error) {
       await setImmediate()
       throw new DstoreError('datastore.query error', error as Error, {
@@ -624,12 +655,21 @@ export class Dstore implements IDstore {
    * It takes a Parameter object and returns an AsyncIterable.
    * Entities returned have been processed by `fixKeys()`.
    * 
+   * Can be used with `for await` loops like this:
+   * 
+   * ```typescript
+   * for await (const entity of dstore.iterate({ kindName: 'p_ReservierungsAbruf', filters: [['verbucht', '=', true]]})) {
+   *   console.log(entity)
+   * }
+   * ```
+   * 
    * @param kindName Name of the [[Datastore]][Kind](https://cloud.google.com/datastore/docs/concepts/entities#kinds_and_identifiers) ("Table") which should be searched.
    * @param filters List of [[Query]] filter() calls.
    * @param limit Maximum Number of Results to return.
    * @param ordering List of [[Query]] order() calls.
    * @param selection selectionList of [[Query]] select() calls.
    *
+   * @throws [[DstoreError]]
    * @category Additional
    */
   async * iterate({
@@ -638,7 +678,7 @@ export class Dstore implements IDstore {
     limit = 2500,
     ordering = [],
     selection = [],
-  }: IIterateParams): AsyncIterable<IDstoreEntry> {
+  }: IIterateParams): AsyncIterable<IDstoreEntryWithKey> {
     assertIsString(kindName)
     assertIsArray(filters)
     assertIsNumber(limit)
@@ -658,10 +698,11 @@ export class Dstore implements IDstore {
       if (selection.length > 0) {
         q.select(selection as any)
       }
-      for await (const entity of this.getDoT().runStream(q)) {
+      for await (const entity of q.runStream()) {
         const ret = this.fixKeys([entity])[0]
         assertIsDefined(ret, 'datastore.iterate: entity is undefined')
-        yield ret
+        assertIsKey(ret[Datastore.KEY])
+        yield ret as IDstoreEntryWithKey
       }
     } catch (error) {
       await setImmediate()