@causa/workspace-google 0.18.0-beta.2 → 0.18.0-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/dist/configurations/generated.d.ts +92 -0
- package/dist/configurations/generated.js +172 -0
- package/dist/configurations/schemas/google.yaml +96 -0
- package/dist/emulators/firebase-storage.d.ts +9 -1
- package/dist/emulators/firebase-storage.js +13 -1
- package/dist/emulators/firestore.d.ts +9 -1
- package/dist/emulators/firestore.js +13 -1
- package/dist/emulators/identity-platform.d.ts +9 -1
- package/dist/emulators/identity-platform.js +13 -1
- package/dist/emulators/pubsub.d.ts +9 -1
- package/dist/emulators/pubsub.js +13 -1
- package/dist/emulators/spanner.d.ts +18 -2
- package/dist/emulators/spanner.js +26 -2
- package/dist/functions/emulator/start-firebase-storage.js +4 -3
- package/dist/functions/emulator/start-firestore.js +5 -10
- package/dist/functions/emulator/start-identity-platform.js +4 -7
- package/dist/functions/emulator/start-pubsub.call.js +10 -9
- package/dist/functions/emulator/start-spanner.call.js +14 -13
- package/dist/functions/event-topic/broker-create-trigger-cloud-run.call.js +12 -4
- package/dist/functions/event-topic/broker-delete-trigger-resource-cloud-run-service.call.d.ts +2 -0
- package/dist/functions/event-topic/broker-delete-trigger-resource-cloud-run-service.call.js +5 -0
- package/dist/functions/event-topic/broker-delete-trigger-resource-cloud-run-service.d.ts +10 -0
- package/dist/functions/event-topic/broker-delete-trigger-resource-cloud-run-service.js +19 -0
- package/dist/functions/event-topic/broker-delete-trigger-resource-pubsub-subscription.js +2 -1
- package/dist/functions/event-topic/broker-publish-events-google.call.js +26 -12
- package/dist/functions/event-topic/broker-wait-for-processing-pubsub.call.d.ts +2 -0
- package/dist/functions/event-topic/broker-wait-for-processing-pubsub.call.js +119 -0
- package/dist/functions/event-topic/broker-wait-for-processing-pubsub.d.ts +16 -0
- package/dist/functions/event-topic/broker-wait-for-processing-pubsub.js +23 -0
- package/dist/functions/event-topic/index.d.ts +2 -0
- package/dist/functions/event-topic/index.js +2 -0
- package/dist/functions/index.js +2 -2
- package/dist/services/cloud-monitoring.d.ts +44 -0
- package/dist/services/cloud-monitoring.js +61 -0
- package/dist/services/cloud-run-pubsub-trigger.d.ts +22 -1
- package/dist/services/cloud-run-pubsub-trigger.js +44 -5
- package/dist/services/cloud-run.d.ts +56 -1
- package/dist/services/cloud-run.js +71 -1
- package/dist/services/cloud-run.utils.d.ts +10 -0
- package/dist/services/cloud-run.utils.js +12 -0
- package/dist/services/google-apis.d.ts +7 -3
- package/dist/services/google-apis.js +1 -2
- package/dist/services/index.d.ts +3 -2
- package/dist/services/index.js +3 -2
- package/dist/services/pubsub.d.ts +1 -0
- package/dist/services/pubsub.js +1 -0
- package/dist/services/pubsub.utils.d.ts +9 -0
- package/dist/services/pubsub.utils.js +11 -0
- package/package.json +14 -14
|
@@ -19,6 +19,18 @@ export function getIdentityPlatformContainerName(context) {
|
|
|
19
19
|
return `${workspaceName}-identity-platform`;
|
|
20
20
|
}
|
|
21
21
|
/**
|
|
22
|
-
* The port on which the Firebase Auth emulator is listening.
|
|
22
|
+
* The default host port on which the Firebase Auth emulator is listening.
|
|
23
23
|
*/
|
|
24
24
|
export const FIREBASE_AUTH_PORT = 9099;
|
|
25
|
+
/**
|
|
26
|
+
* Returns the host port to which the Identity Platform (Firebase Auth) emulator should be bound.
|
|
27
|
+
* Reads `google.identityPlatform.emulator.port`, falling back to {@link FIREBASE_AUTH_PORT}.
|
|
28
|
+
*
|
|
29
|
+
* @param context The {@link WorkspaceContext}.
|
|
30
|
+
* @returns The host port for the Identity Platform emulator.
|
|
31
|
+
*/
|
|
32
|
+
export function getIdentityPlatformEmulatorPort(context) {
|
|
33
|
+
return (context
|
|
34
|
+
.asConfiguration()
|
|
35
|
+
.get('google.identityPlatform.emulator.port') ?? FIREBASE_AUTH_PORT);
|
|
36
|
+
}
|
|
@@ -11,9 +11,17 @@ export declare const PUBSUB_EMULATOR_NAME = "google.pubSub";
|
|
|
11
11
|
*/
|
|
12
12
|
export declare function getPubSubContainerName(context: WorkspaceContext): string;
|
|
13
13
|
/**
|
|
14
|
-
* The port on which the Pub/Sub emulator is listening.
|
|
14
|
+
* The default host port on which the Pub/Sub emulator is listening.
|
|
15
15
|
*/
|
|
16
16
|
export declare const PUBSUB_PORT = 8085;
|
|
17
|
+
/**
|
|
18
|
+
* Returns the host port to which the Pub/Sub emulator should be bound.
|
|
19
|
+
* Reads `google.pubSub.emulator.port`, falling back to {@link PUBSUB_PORT}.
|
|
20
|
+
*
|
|
21
|
+
* @param context The {@link WorkspaceContext}.
|
|
22
|
+
* @returns The host port for the Pub/Sub emulator.
|
|
23
|
+
*/
|
|
24
|
+
export declare function getPubSubEmulatorPort(context: WorkspaceContext): number;
|
|
17
25
|
/**
|
|
18
26
|
* Formats the name of a Pub/Sub topic as an environment variable.
|
|
19
27
|
* The value of the corresponding environment variable is expected to be the Pub/Sub topic ID, e.g.
|
package/dist/emulators/pubsub.js
CHANGED
|
@@ -19,9 +19,21 @@ export function getPubSubContainerName(context) {
|
|
|
19
19
|
return `${workspaceName}-pubsub`;
|
|
20
20
|
}
|
|
21
21
|
/**
|
|
22
|
-
* The port on which the Pub/Sub emulator is listening.
|
|
22
|
+
* The default host port on which the Pub/Sub emulator is listening.
|
|
23
23
|
*/
|
|
24
24
|
export const PUBSUB_PORT = 8085;
|
|
25
|
+
/**
|
|
26
|
+
* Returns the host port to which the Pub/Sub emulator should be bound.
|
|
27
|
+
* Reads `google.pubSub.emulator.port`, falling back to {@link PUBSUB_PORT}.
|
|
28
|
+
*
|
|
29
|
+
* @param context The {@link WorkspaceContext}.
|
|
30
|
+
* @returns The host port for the Pub/Sub emulator.
|
|
31
|
+
*/
|
|
32
|
+
export function getPubSubEmulatorPort(context) {
|
|
33
|
+
return (context
|
|
34
|
+
.asConfiguration()
|
|
35
|
+
.get('google.pubSub.emulator.port') ?? PUBSUB_PORT);
|
|
36
|
+
}
|
|
25
37
|
/**
|
|
26
38
|
* Formats the name of a Pub/Sub topic as an environment variable.
|
|
27
39
|
* The value of the corresponding environment variable is expected to be the Pub/Sub topic ID, e.g.
|
|
@@ -15,10 +15,26 @@ export declare function getSpannerContainerName(context: WorkspaceContext): stri
|
|
|
15
15
|
*/
|
|
16
16
|
export declare const SPANNER_IMAGE = "gcr.io/cloud-spanner-emulator/emulator";
|
|
17
17
|
/**
|
|
18
|
-
* The port on which the Spanner emulator exposes its gRPC API.
|
|
18
|
+
* The default host port on which the Spanner emulator exposes its gRPC API.
|
|
19
19
|
*/
|
|
20
20
|
export declare const SPANNER_GRPC_PORT = 9010;
|
|
21
21
|
/**
|
|
22
|
-
* The port on which the Spanner emulator exposes its HTTP API.
|
|
22
|
+
* The default host port on which the Spanner emulator exposes its HTTP API.
|
|
23
23
|
*/
|
|
24
24
|
export declare const SPANNER_HTTP_PORT = 9020;
|
|
25
|
+
/**
|
|
26
|
+
* Returns the host port to which the Spanner emulator gRPC API should be bound.
|
|
27
|
+
* Reads `google.spanner.emulator.grpcPort`, falling back to {@link SPANNER_GRPC_PORT}.
|
|
28
|
+
*
|
|
29
|
+
* @param context The {@link WorkspaceContext}.
|
|
30
|
+
* @returns The host gRPC port for the Spanner emulator.
|
|
31
|
+
*/
|
|
32
|
+
export declare function getSpannerEmulatorGrpcPort(context: WorkspaceContext): number;
|
|
33
|
+
/**
|
|
34
|
+
* Returns the host port to which the Spanner emulator HTTP API should be bound.
|
|
35
|
+
* Reads `google.spanner.emulator.httpPort`, falling back to {@link SPANNER_HTTP_PORT}.
|
|
36
|
+
*
|
|
37
|
+
* @param context The {@link WorkspaceContext}.
|
|
38
|
+
* @returns The host HTTP port for the Spanner emulator.
|
|
39
|
+
*/
|
|
40
|
+
export declare function getSpannerEmulatorHttpPort(context: WorkspaceContext): number;
|
|
@@ -23,10 +23,34 @@ export function getSpannerContainerName(context) {
|
|
|
23
23
|
*/
|
|
24
24
|
export const SPANNER_IMAGE = `gcr.io/cloud-spanner-emulator/emulator`;
|
|
25
25
|
/**
|
|
26
|
-
* The port on which the Spanner emulator exposes its gRPC API.
|
|
26
|
+
* The default host port on which the Spanner emulator exposes its gRPC API.
|
|
27
27
|
*/
|
|
28
28
|
export const SPANNER_GRPC_PORT = 9010;
|
|
29
29
|
/**
|
|
30
|
-
* The port on which the Spanner emulator exposes its HTTP API.
|
|
30
|
+
* The default host port on which the Spanner emulator exposes its HTTP API.
|
|
31
31
|
*/
|
|
32
32
|
export const SPANNER_HTTP_PORT = 9020;
|
|
33
|
+
/**
|
|
34
|
+
* Returns the host port to which the Spanner emulator gRPC API should be bound.
|
|
35
|
+
* Reads `google.spanner.emulator.grpcPort`, falling back to {@link SPANNER_GRPC_PORT}.
|
|
36
|
+
*
|
|
37
|
+
* @param context The {@link WorkspaceContext}.
|
|
38
|
+
* @returns The host gRPC port for the Spanner emulator.
|
|
39
|
+
*/
|
|
40
|
+
export function getSpannerEmulatorGrpcPort(context) {
|
|
41
|
+
return (context
|
|
42
|
+
.asConfiguration()
|
|
43
|
+
.get('google.spanner.emulator.grpcPort') ?? SPANNER_GRPC_PORT);
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Returns the host port to which the Spanner emulator HTTP API should be bound.
|
|
47
|
+
* Reads `google.spanner.emulator.httpPort`, falling back to {@link SPANNER_HTTP_PORT}.
|
|
48
|
+
*
|
|
49
|
+
* @param context The {@link WorkspaceContext}.
|
|
50
|
+
* @returns The host HTTP port for the Spanner emulator.
|
|
51
|
+
*/
|
|
52
|
+
export function getSpannerEmulatorHttpPort(context) {
|
|
53
|
+
return (context
|
|
54
|
+
.asConfiguration()
|
|
55
|
+
.get('google.spanner.emulator.httpPort') ?? SPANNER_HTTP_PORT);
|
|
56
|
+
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { EmulatorStart } from '@causa/workspace-core';
|
|
2
2
|
import { fileURLToPath } from 'url';
|
|
3
|
-
import { FIREBASE_CONTAINER_STORAGE_RULES_FILE, FIREBASE_STORAGE_EMULATOR_NAME, FIREBASE_STORAGE_PORT, getFirebaseStorageContainerName, } from '../../emulators/index.js';
|
|
3
|
+
import { FIREBASE_CONTAINER_STORAGE_RULES_FILE, FIREBASE_STORAGE_EMULATOR_NAME, FIREBASE_STORAGE_PORT, getFirebaseStorageContainerName, getFirebaseStorageEmulatorPort, } from '../../emulators/index.js';
|
|
4
4
|
import { FirebaseEmulatorService } from '../../services/firebase-emulator.js';
|
|
5
5
|
import { GoogleFirebaseStorageMergeRules } from '../google-firebase-storage/index.js';
|
|
6
6
|
/**
|
|
@@ -30,12 +30,13 @@ export class EmulatorStartForFirebaseStorage extends EmulatorStart {
|
|
|
30
30
|
const { securityRuleFile } = await this._context.call(GoogleFirebaseStorageMergeRules, {});
|
|
31
31
|
this._context.logger.info('️🍱 Starting Firebase Storage emulator.');
|
|
32
32
|
const containerName = getFirebaseStorageContainerName(this._context);
|
|
33
|
+
const hostPort = getFirebaseStorageEmulatorPort(this._context);
|
|
33
34
|
const firebaseEmulatorService = this._context.service(FirebaseEmulatorService);
|
|
34
35
|
await firebaseEmulatorService.start(containerName, FIREBASE_CONF_FILE, [
|
|
35
36
|
{
|
|
36
37
|
host: '127.0.0.1',
|
|
37
38
|
container: FIREBASE_STORAGE_PORT,
|
|
38
|
-
local:
|
|
39
|
+
local: hostPort,
|
|
39
40
|
},
|
|
40
41
|
], {
|
|
41
42
|
mounts: [
|
|
@@ -51,7 +52,7 @@ export class EmulatorStartForFirebaseStorage extends EmulatorStart {
|
|
|
51
52
|
});
|
|
52
53
|
this._context.logger.info('️🍱 Successfully initialized Firebase Storage emulator.');
|
|
53
54
|
return {
|
|
54
|
-
FIREBASE_STORAGE_EMULATOR_HOST: `127.0.0.1:${
|
|
55
|
+
FIREBASE_STORAGE_EMULATOR_HOST: `127.0.0.1:${hostPort}`,
|
|
55
56
|
GOOGLE_CLOUD_PROJECT: firebaseEmulatorService.localGcpProject,
|
|
56
57
|
GCP_PROJECT: firebaseEmulatorService.localGcpProject,
|
|
57
58
|
GCLOUD_PROJECT: firebaseEmulatorService.localGcpProject,
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { EmulatorStart } from '@causa/workspace-core';
|
|
2
|
-
import { FIRESTORE_CONTAINER_RULES_FILE, FIRESTORE_EMULATOR_NAME, FIRESTORE_PORT, getFirestoreContainerName, } from '../../emulators/index.js';
|
|
2
|
+
import { FIRESTORE_CONTAINER_RULES_FILE, FIRESTORE_EMULATOR_NAME, FIRESTORE_PORT, getFirestoreContainerName, getFirestoreEmulatorPort, } from '../../emulators/index.js';
|
|
3
3
|
import { GcloudEmulatorService } from '../../services/gcloud-emulator.js';
|
|
4
4
|
import { GoogleFirestoreMergeRules } from '../google-firestore/index.js';
|
|
5
5
|
/**
|
|
@@ -26,14 +26,9 @@ export class EmulatorStartForFirestore extends EmulatorStart {
|
|
|
26
26
|
const { securityRuleFile } = await this._context.call(GoogleFirestoreMergeRules, {});
|
|
27
27
|
this._context.logger.info('🗃️ Starting Firestore emulator.');
|
|
28
28
|
const containerName = getFirestoreContainerName(this._context);
|
|
29
|
+
const hostPort = getFirestoreEmulatorPort(this._context);
|
|
29
30
|
const gcloudEmulatorService = this._context.service(GcloudEmulatorService);
|
|
30
|
-
await gcloudEmulatorService.start('firestore', containerName, [
|
|
31
|
-
{
|
|
32
|
-
host: '127.0.0.1',
|
|
33
|
-
local: FIRESTORE_PORT,
|
|
34
|
-
container: FIRESTORE_PORT,
|
|
35
|
-
},
|
|
36
|
-
], {
|
|
31
|
+
await gcloudEmulatorService.start('firestore', containerName, [{ host: '127.0.0.1', local: hostPort, container: FIRESTORE_PORT }], {
|
|
37
32
|
mounts: [
|
|
38
33
|
{
|
|
39
34
|
type: 'bind',
|
|
@@ -43,11 +38,11 @@ export class EmulatorStartForFirestore extends EmulatorStart {
|
|
|
43
38
|
},
|
|
44
39
|
],
|
|
45
40
|
additionalArguments: ['--rules', FIRESTORE_CONTAINER_RULES_FILE],
|
|
46
|
-
availabilityEndpoint: `http://127.0.0.1:${
|
|
41
|
+
availabilityEndpoint: `http://127.0.0.1:${hostPort}/`,
|
|
47
42
|
});
|
|
48
43
|
this._context.logger.info('🗃️ Successfully initialized Firestore emulator.');
|
|
49
44
|
return {
|
|
50
|
-
FIRESTORE_EMULATOR_HOST: `127.0.0.1:${
|
|
45
|
+
FIRESTORE_EMULATOR_HOST: `127.0.0.1:${hostPort}`,
|
|
51
46
|
GOOGLE_CLOUD_PROJECT: gcloudEmulatorService.localGcpProject,
|
|
52
47
|
GCP_PROJECT: gcloudEmulatorService.localGcpProject,
|
|
53
48
|
GCLOUD_PROJECT: gcloudEmulatorService.localGcpProject,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { EmulatorStart } from '@causa/workspace-core';
|
|
2
2
|
import { fileURLToPath } from 'url';
|
|
3
|
-
import { FIREBASE_AUTH_PORT, IDENTITY_PLATFORM_EMULATOR_NAME, getIdentityPlatformContainerName, } from '../../emulators/index.js';
|
|
3
|
+
import { FIREBASE_AUTH_PORT, IDENTITY_PLATFORM_EMULATOR_NAME, getIdentityPlatformContainerName, getIdentityPlatformEmulatorPort, } from '../../emulators/index.js';
|
|
4
4
|
import { FirebaseEmulatorService } from '../../services/firebase-emulator.js';
|
|
5
5
|
/**
|
|
6
6
|
* The Firebase configuration file enabling only the Auth emulator.
|
|
@@ -27,17 +27,14 @@ export class EmulatorStartForIdentityPlatform extends EmulatorStart {
|
|
|
27
27
|
}
|
|
28
28
|
this._context.logger.info('🛂 Starting the Identity Platform emulator.');
|
|
29
29
|
const containerName = getIdentityPlatformContainerName(this._context);
|
|
30
|
+
const hostPort = getIdentityPlatformEmulatorPort(this._context);
|
|
30
31
|
const firebaseEmulatorService = this._context.service(FirebaseEmulatorService);
|
|
31
32
|
await firebaseEmulatorService.start(containerName, FIREBASE_CONF_FILE, [
|
|
32
|
-
{
|
|
33
|
-
host: '127.0.0.1',
|
|
34
|
-
local: FIREBASE_AUTH_PORT,
|
|
35
|
-
container: FIREBASE_AUTH_PORT,
|
|
36
|
-
},
|
|
33
|
+
{ host: '127.0.0.1', local: hostPort, container: FIREBASE_AUTH_PORT },
|
|
37
34
|
]);
|
|
38
35
|
this._context.logger.info('🛂 Successfully initialized the Identity Platform emulator.');
|
|
39
36
|
return {
|
|
40
|
-
FIREBASE_AUTH_EMULATOR_HOST: `127.0.0.1:${
|
|
37
|
+
FIREBASE_AUTH_EMULATOR_HOST: `127.0.0.1:${hostPort}`,
|
|
41
38
|
GOOGLE_CLOUD_PROJECT: firebaseEmulatorService.localGcpProject,
|
|
42
39
|
GCP_PROJECT: firebaseEmulatorService.localGcpProject,
|
|
43
40
|
GCLOUD_PROJECT: firebaseEmulatorService.localGcpProject,
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { EventTopicList, } from '@causa/workspace-core';
|
|
2
2
|
import { PubSub } from '@google-cloud/pubsub';
|
|
3
3
|
import { getLocalGcpProject } from '../../configurations/utils.js';
|
|
4
|
-
import { PUBSUB_EMULATOR_NAME, PUBSUB_PORT, formatPubSubTopicAsEnvironmentVariable, getPubSubContainerName, } from '../../emulators/index.js';
|
|
4
|
+
import { PUBSUB_EMULATOR_NAME, PUBSUB_PORT, formatPubSubTopicAsEnvironmentVariable, getPubSubContainerName, getPubSubEmulatorPort, } from '../../emulators/index.js';
|
|
5
5
|
import { GcloudEmulatorService } from '../../services/gcloud-emulator.js';
|
|
6
6
|
export default async function call() {
|
|
7
7
|
const configuration = await startPubSub(this, this._context);
|
|
@@ -11,27 +11,28 @@ async function startPubSub(self, context) {
|
|
|
11
11
|
if (self.dryRun) {
|
|
12
12
|
return {};
|
|
13
13
|
}
|
|
14
|
-
const
|
|
15
|
-
const
|
|
14
|
+
const hostPort = getPubSubEmulatorPort(context);
|
|
15
|
+
const emulatorConf = await startPubSubEmulator(context, hostPort);
|
|
16
|
+
const topicsConf = await createTopicsIfNeeded(context, hostPort);
|
|
16
17
|
context.logger.info('📫 Successfully initialized Pub/Sub emulator.');
|
|
17
18
|
return { ...emulatorConf, ...topicsConf };
|
|
18
19
|
}
|
|
19
|
-
async function startPubSubEmulator(context) {
|
|
20
|
+
async function startPubSubEmulator(context, hostPort) {
|
|
20
21
|
context.logger.info('📫 Starting Pub/Sub emulator.');
|
|
21
22
|
const gcpProject = getLocalGcpProject(context);
|
|
22
23
|
const containerName = getPubSubContainerName(context);
|
|
23
24
|
const gcloudEmulatorService = context.service(GcloudEmulatorService);
|
|
24
|
-
await gcloudEmulatorService.start('pubsub', containerName, [{ host: '127.0.0.1', local:
|
|
25
|
-
availabilityEndpoint: `http://127.0.0.1:${
|
|
25
|
+
await gcloudEmulatorService.start('pubsub', containerName, [{ host: '127.0.0.1', local: hostPort, container: PUBSUB_PORT }], {
|
|
26
|
+
availabilityEndpoint: `http://127.0.0.1:${hostPort}/v1/projects/${gcpProject}/topics`,
|
|
26
27
|
});
|
|
27
28
|
return {
|
|
28
|
-
PUBSUB_EMULATOR_HOST: `127.0.0.1:${
|
|
29
|
+
PUBSUB_EMULATOR_HOST: `127.0.0.1:${hostPort}`,
|
|
29
30
|
GOOGLE_CLOUD_PROJECT: gcloudEmulatorService.localGcpProject,
|
|
30
31
|
GCP_PROJECT: gcloudEmulatorService.localGcpProject,
|
|
31
32
|
GCLOUD_PROJECT: gcloudEmulatorService.localGcpProject,
|
|
32
33
|
};
|
|
33
34
|
}
|
|
34
|
-
async function createTopicsIfNeeded(context) {
|
|
35
|
+
async function createTopicsIfNeeded(context, hostPort) {
|
|
35
36
|
if (context.get('events.broker') !== 'google.pubSub') {
|
|
36
37
|
context.logger.warn('📫️ Pub/Sub is not set as the event broker. Skipping event topics setup.');
|
|
37
38
|
return {};
|
|
@@ -40,7 +41,7 @@ async function createTopicsIfNeeded(context) {
|
|
|
40
41
|
const gcpProject = getLocalGcpProject(context);
|
|
41
42
|
const pubSub = new PubSub({
|
|
42
43
|
projectId: gcpProject,
|
|
43
|
-
apiEndpoint: `127.0.0.1:${
|
|
44
|
+
apiEndpoint: `127.0.0.1:${hostPort}`,
|
|
44
45
|
});
|
|
45
46
|
const envVars = await Promise.all(topics.map(async ({ id }) => {
|
|
46
47
|
context.logger.info(`📫 Creating Pub/Sub emulator topic '${id}'.`);
|
|
@@ -5,7 +5,7 @@ import { Instance, Spanner } from '@google-cloud/spanner';
|
|
|
5
5
|
// version can be loaded.
|
|
6
6
|
import { grpc } from 'google-gax';
|
|
7
7
|
import { getLocalGcpProject, } from '../../configurations/index.js';
|
|
8
|
-
import { SPANNER_EMULATOR_NAME, SPANNER_GRPC_PORT, SPANNER_HTTP_PORT, SPANNER_IMAGE, getSpannerContainerName, } from '../../emulators/index.js';
|
|
8
|
+
import { SPANNER_EMULATOR_NAME, SPANNER_GRPC_PORT, SPANNER_HTTP_PORT, SPANNER_IMAGE, getSpannerContainerName, getSpannerEmulatorGrpcPort, getSpannerEmulatorHttpPort, } from '../../emulators/index.js';
|
|
9
9
|
import { GoogleSpannerListDatabases } from '../google-spanner/index.js';
|
|
10
10
|
export default async function call() {
|
|
11
11
|
const configuration = await startSpanner(this, this._context);
|
|
@@ -15,12 +15,14 @@ async function startSpanner(self, context) {
|
|
|
15
15
|
if (self.dryRun) {
|
|
16
16
|
return {};
|
|
17
17
|
}
|
|
18
|
-
const
|
|
19
|
-
const
|
|
18
|
+
const grpcHostPort = getSpannerEmulatorGrpcPort(context);
|
|
19
|
+
const httpHostPort = getSpannerEmulatorHttpPort(context);
|
|
20
|
+
const emulatorConf = await startSpannerEmulator(context, grpcHostPort, httpHostPort);
|
|
21
|
+
const instanceAndDatabaseConf = await initializeEmulator(context, grpcHostPort);
|
|
20
22
|
context.logger.info('🗃️ Successfully initialized Spanner emulator.');
|
|
21
23
|
return { ...emulatorConf, ...instanceAndDatabaseConf };
|
|
22
24
|
}
|
|
23
|
-
async function startSpannerEmulator(context) {
|
|
25
|
+
async function startSpannerEmulator(context, grpcHostPort, httpHostPort) {
|
|
24
26
|
context.logger.info('🗃️ Starting Spanner emulator.');
|
|
25
27
|
const gcpProject = getLocalGcpProject(context);
|
|
26
28
|
const containerName = getSpannerContainerName(context);
|
|
@@ -28,23 +30,22 @@ async function startSpannerEmulator(context) {
|
|
|
28
30
|
.asConfiguration()
|
|
29
31
|
.get('google.spanner.emulator.version') ?? 'latest';
|
|
30
32
|
const dockerEmulatorService = context.service(DockerEmulatorService);
|
|
31
|
-
await dockerEmulatorService.start(`${SPANNER_IMAGE}:${imageVersion}`, containerName, [
|
|
32
|
-
host: '127.0.0.1',
|
|
33
|
-
local:
|
|
34
|
-
|
|
35
|
-
})
|
|
36
|
-
await dockerEmulatorService.waitForAvailability(containerName, `http://127.0.0.1:${SPANNER_HTTP_PORT}/v1/projects/${gcpProject}/instances`);
|
|
33
|
+
await dockerEmulatorService.start(`${SPANNER_IMAGE}:${imageVersion}`, containerName, [
|
|
34
|
+
{ host: '127.0.0.1', local: grpcHostPort, container: SPANNER_GRPC_PORT },
|
|
35
|
+
{ host: '127.0.0.1', local: httpHostPort, container: SPANNER_HTTP_PORT },
|
|
36
|
+
]);
|
|
37
|
+
await dockerEmulatorService.waitForAvailability(containerName, `http://127.0.0.1:${httpHostPort}/v1/projects/${gcpProject}/instances`);
|
|
37
38
|
return {
|
|
38
|
-
SPANNER_EMULATOR_HOST: `127.0.0.1:${
|
|
39
|
+
SPANNER_EMULATOR_HOST: `127.0.0.1:${grpcHostPort}`,
|
|
39
40
|
GOOGLE_CLOUD_PROJECT: gcpProject,
|
|
40
41
|
GCP_PROJECT: gcpProject,
|
|
41
42
|
GCLOUD_PROJECT: gcpProject,
|
|
42
43
|
};
|
|
43
44
|
}
|
|
44
|
-
async function initializeEmulator(context) {
|
|
45
|
+
async function initializeEmulator(context, port) {
|
|
45
46
|
const spanner = new Spanner({
|
|
46
47
|
servicePath: '127.0.0.1',
|
|
47
|
-
port
|
|
48
|
+
port,
|
|
48
49
|
projectId: getLocalGcpProject(context),
|
|
49
50
|
sslCreds: grpc.credentials.createInsecure(),
|
|
50
51
|
});
|
|
@@ -45,10 +45,18 @@ function resolveFromObject(context, trigger) {
|
|
|
45
45
|
};
|
|
46
46
|
}
|
|
47
47
|
export default async function call() {
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
48
|
+
// Cloning is only supported for project-scoped triggers, where the service is defined by the project's configuration.
|
|
49
|
+
// Raw URI triggers keep targeting the live service.
|
|
50
|
+
const isProjectScoped = typeof this.trigger !== 'string';
|
|
51
|
+
const { serviceId, path } = isProjectScoped
|
|
52
|
+
? resolveFromObject(this._context, this.trigger)
|
|
53
|
+
: resolveFromString(this._context, this.trigger);
|
|
54
|
+
const clone = isProjectScoped
|
|
55
|
+
? this._context
|
|
56
|
+
.asConfiguration()
|
|
57
|
+
.get('google.cloudRun.eventBackfillServiceCloneConfig')
|
|
58
|
+
: undefined;
|
|
51
59
|
return await this._context
|
|
52
60
|
.service(CloudRunPubSubTriggerService)
|
|
53
|
-
.create(this.backfillId, this.topicId, serviceId, path);
|
|
61
|
+
.create(this.backfillId, this.topicId, serviceId, path, { clone });
|
|
54
62
|
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { EventTopicBrokerDeleteTriggerResource } from '@causa/workspace-core';
|
|
2
|
+
/**
|
|
3
|
+
* Implements {@link EventTopicBrokerDeleteTriggerResource} for Cloud Run services.
|
|
4
|
+
* When `google.cloudRun.eventBackfillServiceCloneConfig` is set, a temporary copy of the Cloud Run service is created
|
|
5
|
+
* for each project-scoped trigger, isolating the backfill load from production. This function deletes that copy.
|
|
6
|
+
*/
|
|
7
|
+
export declare class EventTopicBrokerDeleteTriggerResourceForCloudRunService extends EventTopicBrokerDeleteTriggerResource {
|
|
8
|
+
_call(): Promise<void>;
|
|
9
|
+
_supports(): boolean;
|
|
10
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { callDeferred } from '@causa/workspace';
|
|
2
|
+
import { EventTopicBrokerDeleteTriggerResource, } from '@causa/workspace-core';
|
|
3
|
+
import { CLOUD_RUN_SERVICE_ID_REGEX } from '../../services/cloud-run.utils.js';
|
|
4
|
+
/**
|
|
5
|
+
* Implements {@link EventTopicBrokerDeleteTriggerResource} for Cloud Run services.
|
|
6
|
+
* When `google.cloudRun.eventBackfillServiceCloneConfig` is set, a temporary copy of the Cloud Run service is created
|
|
7
|
+
* for each project-scoped trigger, isolating the backfill load from production. This function deletes that copy.
|
|
8
|
+
*/
|
|
9
|
+
export class EventTopicBrokerDeleteTriggerResourceForCloudRunService extends EventTopicBrokerDeleteTriggerResource {
|
|
10
|
+
async _call() {
|
|
11
|
+
return await callDeferred(this, import.meta.url);
|
|
12
|
+
}
|
|
13
|
+
_supports() {
|
|
14
|
+
return (CLOUD_RUN_SERVICE_ID_REGEX.test(this.id) &&
|
|
15
|
+
this._context
|
|
16
|
+
.asConfiguration()
|
|
17
|
+
.get('events.broker') === 'google.pubSub');
|
|
18
|
+
}
|
|
19
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { callDeferred } from '@causa/workspace';
|
|
2
2
|
import { EventTopicBrokerDeleteTriggerResource, } from '@causa/workspace-core';
|
|
3
|
+
import { PUBSUB_SUBSCRIPTION_ID_REGEX } from '../../services/pubsub.utils.js';
|
|
3
4
|
/**
|
|
4
5
|
* Implements {@link EventTopicBrokerDeleteTriggerResource} for Pub/Sub subscriptions.
|
|
5
6
|
* Pub/Sub subscriptions are for example used when creating Cloud Run triggers.
|
|
@@ -9,7 +10,7 @@ export class EventTopicBrokerDeleteTriggerResourceForPubSubSubscription extends
|
|
|
9
10
|
return await callDeferred(this, import.meta.url);
|
|
10
11
|
}
|
|
11
12
|
_supports() {
|
|
12
|
-
return (this.id
|
|
13
|
+
return (PUBSUB_SUBSCRIPTION_ID_REGEX.test(this.id) &&
|
|
13
14
|
this._context
|
|
14
15
|
.asConfiguration()
|
|
15
16
|
.get('events.broker') === 'google.pubSub');
|
|
@@ -5,23 +5,37 @@ import { PubSubService } from '../../services/index.js';
|
|
|
5
5
|
*/
|
|
6
6
|
const PUBLISH_PROGRESS_LOG_INTERVAL = 10000;
|
|
7
7
|
/**
|
|
8
|
-
*
|
|
8
|
+
* Default options when publishing events in batches to Pub/Sub.
|
|
9
|
+
* These can be overridden through the `google.pubSub.backfillPublishOptions` configuration.
|
|
9
10
|
*/
|
|
10
|
-
const
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
maxBytes: 10 * 1024 * 1024,
|
|
17
|
-
maxMessages: 1000,
|
|
18
|
-
maxMilliseconds: 1000,
|
|
19
|
-
},
|
|
11
|
+
const DEFAULT_PUBLISH_OPTIONS = {
|
|
12
|
+
maxOutstandingBytes: 10 * 1024 * 1024,
|
|
13
|
+
maxOutstandingMessages: 1000,
|
|
14
|
+
maxBytes: 10 * 1024 * 1024,
|
|
15
|
+
maxMessages: 1000,
|
|
16
|
+
maxMilliseconds: 1000,
|
|
20
17
|
};
|
|
21
18
|
export default async function call() {
|
|
19
|
+
const publishOptions = {
|
|
20
|
+
...DEFAULT_PUBLISH_OPTIONS,
|
|
21
|
+
...this._context
|
|
22
|
+
.asConfiguration()
|
|
23
|
+
.get('google.pubSub.backfillPublishOptions'),
|
|
24
|
+
};
|
|
25
|
+
const options = {
|
|
26
|
+
flowControlOptions: {
|
|
27
|
+
maxOutstandingBytes: publishOptions.maxOutstandingBytes,
|
|
28
|
+
maxOutstandingMessages: publishOptions.maxOutstandingMessages,
|
|
29
|
+
},
|
|
30
|
+
batching: {
|
|
31
|
+
maxBytes: publishOptions.maxBytes,
|
|
32
|
+
maxMessages: publishOptions.maxMessages,
|
|
33
|
+
maxMilliseconds: publishOptions.maxMilliseconds,
|
|
34
|
+
},
|
|
35
|
+
};
|
|
22
36
|
const publisher = this._context
|
|
23
37
|
.service(PubSubService)
|
|
24
|
-
.pubSub.topic(this.topicId,
|
|
38
|
+
.pubSub.topic(this.topicId, options)
|
|
25
39
|
.flowControlled();
|
|
26
40
|
this._context.logger.info('📫 Publishing events.');
|
|
27
41
|
let numEvents = 0;
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import { setTimeout } from 'timers/promises';
|
|
2
|
+
import { CloudMonitoringService, } from '../../services/cloud-monitoring.js';
|
|
3
|
+
import { PUBSUB_SUBSCRIPTION_ID_REGEX, shortSubscriptionId, } from '../../services/index.js';
|
|
4
|
+
/**
|
|
5
|
+
* The interval between two consecutive polls of the Cloud Monitoring API, in milliseconds.
|
|
6
|
+
*/
|
|
7
|
+
const POLL_INTERVAL = 30 * 1000;
|
|
8
|
+
/**
|
|
9
|
+
* The duration of the trailing window of points fetched on each poll, in milliseconds.
|
|
10
|
+
* It must be greater than `QUIET_WINDOW` plus a margin for the Cloud Monitoring export lag.
|
|
11
|
+
*/
|
|
12
|
+
const TRAILING_WINDOW = 10 * 60 * 1000;
|
|
13
|
+
/**
|
|
14
|
+
* The minimum duration (server-time, derived from point timestamps) over which all three monitored metrics must
|
|
15
|
+
* continuously report zero before a subscription is considered drained, in milliseconds.
|
|
16
|
+
*/
|
|
17
|
+
const QUIET_WINDOW = 3 * 60 * 1000;
|
|
18
|
+
/**
|
|
19
|
+
* The alignment period used when querying the cumulative `ack_message_count` metric so that points represent the
|
|
20
|
+
* number of acks within each bucket (delta), rather than the absolute cumulative value, in seconds.
|
|
21
|
+
*/
|
|
22
|
+
const ALIGNMENT_PERIOD = 60;
|
|
23
|
+
/**
|
|
24
|
+
* The overall timeout for waiting on a single subscription, in milliseconds.
|
|
25
|
+
* On timeout the function throws so that the backfill file is preserved for manual cleanup.
|
|
26
|
+
*/
|
|
27
|
+
const TIMEOUT = 60 * 60 * 1000;
|
|
28
|
+
/**
|
|
29
|
+
* Returns the Cloud Monitoring filter for the given metric on the given subscription.
|
|
30
|
+
*/
|
|
31
|
+
function buildFilter(metricType, subscriptionId) {
|
|
32
|
+
return [
|
|
33
|
+
`metric.type = "${metricType}"`,
|
|
34
|
+
`resource.type = "pubsub_subscription"`,
|
|
35
|
+
`resource.labels.subscription_id = "${subscriptionId}"`,
|
|
36
|
+
].join(' AND ');
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Checks whether a gauge metric has been quiet (zero-valued) for at least {@link QUIET_WINDOW}.
|
|
40
|
+
*
|
|
41
|
+
* The check is purely server-side: the latest zero-valued point's timestamp must be at least `QUIET_WINDOW` after the
|
|
42
|
+
* latest non-zero point's timestamp. When no non-zero point exists in the trailing window, the metric is considered
|
|
43
|
+
* quiet (the outer liftoff gate is responsible for distinguishing "cold start" from "already drained").
|
|
44
|
+
*/
|
|
45
|
+
function isGaugeQuiet(points) {
|
|
46
|
+
const lastNonZeroTimestamp = points.filter(({ value }) => value > 0).at(-1)?.timestamp ?? -Infinity;
|
|
47
|
+
const latestZeroTimestamp = points.filter(({ value }) => value === 0).at(-1)?.timestamp ?? -Infinity;
|
|
48
|
+
return latestZeroTimestamp - lastNonZeroTimestamp >= QUIET_WINDOW;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Checks whether a cumulative metric (queried with `ALIGN_DELTA`) has had no non-zero activity in the trailing
|
|
52
|
+
* `QUIET_WINDOW` before the given reference time.
|
|
53
|
+
*
|
|
54
|
+
* Unlike a gauge, Pub/Sub stops writing samples for a cumulative metric while there is no activity. The server-side
|
|
55
|
+
* "still has zero points" anchor used by {@link isGaugeQuiet} therefore does not apply: the absence of points must be
|
|
56
|
+
* interpreted against an external time reference (the query end).
|
|
57
|
+
*/
|
|
58
|
+
function isCumulativeQuiet(points, referenceTime) {
|
|
59
|
+
const lastNonZeroTimestamp = points.filter(({ value }) => value > 0).at(-1)?.timestamp ?? -Infinity;
|
|
60
|
+
return referenceTime.getTime() - lastNonZeroTimestamp >= QUIET_WINDOW;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Observes a single subscription by fetching its three metrics over the configured trailing window and evaluating the
|
|
64
|
+
* liftoff and quiet conditions.
|
|
65
|
+
*/
|
|
66
|
+
async function observeSubscription(context, subscriptionId) {
|
|
67
|
+
const cms = context.service(CloudMonitoringService);
|
|
68
|
+
const end = new Date();
|
|
69
|
+
const start = new Date(end.getTime() - TRAILING_WINDOW);
|
|
70
|
+
const ackFilter = buildFilter('pubsub.googleapis.com/subscription/ack_message_count', subscriptionId);
|
|
71
|
+
const backlogFilter = buildFilter('pubsub.googleapis.com/subscription/num_undelivered_messages', subscriptionId);
|
|
72
|
+
const ageFilter = buildFilter('pubsub.googleapis.com/subscription/oldest_unacked_message_age', subscriptionId);
|
|
73
|
+
const [ackPoints, backlogPoints, agePoints] = await Promise.all([
|
|
74
|
+
cms.listPoints(ackFilter, start, end, {
|
|
75
|
+
aligner: 'ALIGN_DELTA',
|
|
76
|
+
alignmentPeriod: ALIGNMENT_PERIOD,
|
|
77
|
+
}),
|
|
78
|
+
cms.listPoints(backlogFilter, start, end),
|
|
79
|
+
cms.listPoints(ageFilter, start, end),
|
|
80
|
+
]);
|
|
81
|
+
const liftoff = ackPoints.some(({ value }) => value > 0);
|
|
82
|
+
const backlog = backlogPoints.at(-1)?.value ?? null;
|
|
83
|
+
const drained = liftoff &&
|
|
84
|
+
isCumulativeQuiet(ackPoints, end) &&
|
|
85
|
+
isGaugeQuiet(backlogPoints) &&
|
|
86
|
+
isGaugeQuiet(agePoints);
|
|
87
|
+
return { liftoff, backlog, drained };
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Waits for a single subscription to be drained, polling Cloud Monitoring until either the drained state is reached
|
|
91
|
+
* or the overall timeout elapses.
|
|
92
|
+
*/
|
|
93
|
+
async function waitForSubscription(context, subscriptionId) {
|
|
94
|
+
const deadline = Date.now() + TIMEOUT;
|
|
95
|
+
while (true) {
|
|
96
|
+
const { drained, backlog, liftoff } = await observeSubscription(context, subscriptionId);
|
|
97
|
+
if (drained) {
|
|
98
|
+
context.logger.debug(`✅ Subscription '${subscriptionId}' drained: backlog is empty and metrics have been quiet for at least ${QUIET_WINDOW / 1000}s.`);
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
const backlogLabel = backlog === null ? 'no data yet' : `backlog=${backlog}`;
|
|
102
|
+
context.logger.debug(`⏳ Subscription '${subscriptionId}' not drained yet (liftoff=${liftoff}, ${backlogLabel}).`);
|
|
103
|
+
if (Date.now() >= deadline) {
|
|
104
|
+
throw new Error(`Timed out after ${Math.round(TIMEOUT / 60_000)} minute(s) waiting for subscription '${subscriptionId}' to drain.`);
|
|
105
|
+
}
|
|
106
|
+
await setTimeout(POLL_INTERVAL);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
export default async function call() {
|
|
110
|
+
const subscriptionIds = this.temporaryData.temporaryTriggerResourceIds
|
|
111
|
+
.filter((id) => PUBSUB_SUBSCRIPTION_ID_REGEX.test(id))
|
|
112
|
+
.map(shortSubscriptionId);
|
|
113
|
+
if (subscriptionIds.length === 0) {
|
|
114
|
+
throw new Error(`Cannot wait for processing of backfill on event topic '${this.eventTopic}': no temporary Pub/Sub subscription was created. Only backfills with temporary triggers are supported.`);
|
|
115
|
+
}
|
|
116
|
+
this._context.logger.info(`⏳ Waiting for processing of ${subscriptionIds.length} temporary Pub/Sub subscription(s) for event topic '${this.eventTopic}'.`);
|
|
117
|
+
await Promise.all(subscriptionIds.map((id) => waitForSubscription(this._context, id)));
|
|
118
|
+
this._context.logger.info(`✅ All temporary Pub/Sub subscriptions for event topic '${this.eventTopic}' have been drained.`);
|
|
119
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { EventTopicBrokerWaitForProcessing } from '@causa/workspace-core';
|
|
2
|
+
/**
|
|
3
|
+
* Implements {@link EventTopicBrokerWaitForProcessing} for Pub/Sub temporary subscriptions.
|
|
4
|
+
*
|
|
5
|
+
* Only backfills that created temporary push subscriptions are supported: at least one entry of
|
|
6
|
+
* {@link BackfillTemporaryData.temporaryTriggerResourceIds} must match a subscription ID. If none does, the function
|
|
7
|
+
* throws — there is no observable signal for backfills that publish into existing topics with existing subscriptions.
|
|
8
|
+
*
|
|
9
|
+
* Completion is decided per-subscription from Cloud Monitoring time series (`subscription/ack_message_count`,
|
|
10
|
+
* `subscription/num_undelivered_messages`, `subscription/oldest_unacked_message_age`). See the call file for the exact
|
|
11
|
+
* decision logic.
|
|
12
|
+
*/
|
|
13
|
+
export declare class EventTopicBrokerWaitForProcessingForPubSub extends EventTopicBrokerWaitForProcessing {
|
|
14
|
+
_call(): Promise<void>;
|
|
15
|
+
_supports(): boolean;
|
|
16
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { callDeferred } from '@causa/workspace';
|
|
2
|
+
import { EventTopicBrokerWaitForProcessing, } from '@causa/workspace-core';
|
|
3
|
+
/**
|
|
4
|
+
* Implements {@link EventTopicBrokerWaitForProcessing} for Pub/Sub temporary subscriptions.
|
|
5
|
+
*
|
|
6
|
+
* Only backfills that created temporary push subscriptions are supported: at least one entry of
|
|
7
|
+
* {@link BackfillTemporaryData.temporaryTriggerResourceIds} must match a subscription ID. If none does, the function
|
|
8
|
+
* throws — there is no observable signal for backfills that publish into existing topics with existing subscriptions.
|
|
9
|
+
*
|
|
10
|
+
* Completion is decided per-subscription from Cloud Monitoring time series (`subscription/ack_message_count`,
|
|
11
|
+
* `subscription/num_undelivered_messages`, `subscription/oldest_unacked_message_age`). See the call file for the exact
|
|
12
|
+
* decision logic.
|
|
13
|
+
*/
|
|
14
|
+
export class EventTopicBrokerWaitForProcessingForPubSub extends EventTopicBrokerWaitForProcessing {
|
|
15
|
+
async _call() {
|
|
16
|
+
return await callDeferred(this, import.meta.url);
|
|
17
|
+
}
|
|
18
|
+
_supports() {
|
|
19
|
+
return (this._context
|
|
20
|
+
.asConfiguration()
|
|
21
|
+
.get('events.broker') === 'google.pubSub');
|
|
22
|
+
}
|
|
23
|
+
}
|