@contember/cli 1.2.0-alpha.8 → 1.2.0-beta.1

package/src/commands/transfer/ExportCommand.ts

@@ -0,0 +1,72 @@
+import { Command, CommandConfiguration, Input, Workspace } from '@contember/cli-common'
+import { createWriteStream } from 'node:fs'
+import { dataExport, resolveProject } from './utils'
+import { maskToken } from '../../utils/token'
+import { pipeline } from 'node:stream/promises'
+import { rewriteStdoutLine } from '../../utils/stdio'
+
+type Args = {
+	source?: string
+}
+type Options = {
+	'include-system'?: boolean
+	'no-gzip'?: boolean
+	output?: string
+}
+
+export class ExportCommand extends Command<Args, Options> {
+	constructor(
+		private readonly workspace: Workspace,
+	) {
+		super()
+	}
+
+	protected configure(configuration: CommandConfiguration<Args, Options>): void {
+		configuration.description('Export data from a project')
+		const from = configuration.argument('source')
+		if (this.workspace.isSingleProjectMode()) {
+			from.optional()
+		}
+		configuration.option('include-system').valueNone()
+		configuration.option('no-gzip').valueNone()
+		configuration.option('output').valueRequired()
+	}
+
+	protected async execute(input: Input<Args, Options>): Promise<void | number> {
+		const from = input.getArgument('source')
+		const project = await resolveProject({ workspace: this.workspace, projectOrDsn: from })
+
+		console.log('')
+		console.log('Exporting data from a following project')
+		console.log('')
+		console.log(`Project name: ${project.project}`)
+		console.log(`API URL: ${project.baseUrl}`)
+		console.log(`Token: ${maskToken(project.token)}`)
+
+		const noGzip = input.getOption('no-gzip') === true
+		const includeSystem = input.getOption('include-system') === true
+
+		const response = await dataExport({ project, includeSystem, noGzip })
+
+		let transferred = 0
+		let start = Date.now()
+		let lastMbReported = 0
+		console.log('')
+		response.on('data', chunk => {
+			transferred += chunk.length
+			const currentMb = Math.floor(transferred / 1024 / 1024)
+			if (currentMb > lastMbReported) {
+				const durationS = Math.floor((Date.now() - start) / 1000)
+				rewriteStdoutLine(`transferred ${currentMb} MiB; ${durationS} seconds`)
+				lastMbReported = currentMb
+			}
+		})
+
+		const fileName = input.getOption('output') ?? `${project.project}.jsonl${noGzip ? '' : '.gz'}`
+		const fileStream = createWriteStream(fileName)
+
+		await pipeline(response, fileStream)
+
+		console.log('\nExport done.')
+	}
+}

package/src/commands/transfer/ImportCommand.ts

@@ -0,0 +1,61 @@
+import { Command, CommandConfiguration, Input, Workspace } from '@contember/cli-common'
+import { createReadStream } from 'node:fs'
+import { createGunzip } from 'node:zlib'
+import { confirmImport, dataImport, resolveProject } from './utils'
+import { maskToken } from '../../utils/token'
+import { rewriteStdoutLine } from '../../utils/stdio'
+import { readStream } from '../../utils/stream'
+
+type Args = {
+	file: string
+	target?: string
+}
+type Options = {
+	'yes': boolean
+}
+
+export class ImportCommand extends Command<Args, Options> {
+	constructor(
+		private readonly workspace: Workspace,
+	) {
+		super()
+	}
+
+	protected configure(configuration: CommandConfiguration<Args, Options>): void {
+		configuration.description('Import data into a project')
+		configuration.argument('file')
+		const from = configuration.argument('target')
+		if (this.workspace.isSingleProjectMode()) {
+			from.optional()
+		}
+		configuration.option('yes').valueNone()
+	}
+
+	protected async execute(input: Input<Args, Options>): Promise<void | number> {
+		const from = input.getArgument('target')
+		const project = await resolveProject({ projectOrDsn: from, workspace: this.workspace })
+		console.log('')
+		console.log('Importing data into a following project')
+		console.log('')
+		console.log(`Project name: ${project.project}`)
+		console.log(`API URL: ${project.baseUrl}`)
+		console.log(`Token: ${maskToken(project.token)}`)
+		console.log('')
+		if (!await confirmImport(input)) {
+			return 1
+		}
+		const file = input.getArgument('file')
+		const baseInputStream = createReadStream(file)
+		const stream = (file.endsWith('.gz') ? baseInputStream.pipe(createGunzip()) : baseInputStream)
+		const response = await dataImport({ stream: stream, project, printProgress: rewriteStdoutLine })
+		console.log('')
+		const responseData = JSON.parse((await readStream(response)).toString())
+		if (responseData.ok) {
+			console.log('Import done.')
+		} else {
+			console.error('Import failed:')
+			console.log(responseData)
+		}
+	}
+}
+

package/src/commands/transfer/TransferCommand.ts

@@ -0,0 +1,71 @@
+import { Command, CommandConfiguration, Input, Workspace } from '@contember/cli-common'
+import { confirmImport, dataExport, dataImport, resolveProject } from './utils'
+import { maskToken } from '../../utils/token'
+import { rewriteStdoutLine } from '../../utils/stdio'
+import { readStream } from '../../utils/stream'
+
+type Args = {
+	source: string
+	target: string
+}
+type Options = {
+	'include-system'?: boolean
+	yes: boolean
+}
+
+export class TransferCommand extends Command<Args, Options> {
+	constructor(
+		private readonly workspace: Workspace,
+	) {
+		super()
+	}
+
+	protected configure(configuration: CommandConfiguration<Args, Options>): void {
+		configuration.description('Transfer data between projects')
+		configuration.argument('source')
+		const target = configuration.argument('target')
+		if (this.workspace.isSingleProjectMode()) {
+			target.optional()
+		}
+		configuration.option('include-system').valueNone()
+		configuration.option('yes').valueNone()
+	}
+
+	protected async execute(input: Input<Args, Options>): Promise<void | number> {
+		const source = input.getArgument('source')
+		const sourceProject = await resolveProject({ projectOrDsn: source, workspace: this.workspace })
+
+		const target = input.getArgument('target')
+		const targetProject = await resolveProject({ projectOrDsn: target, workspace: this.workspace })
+
+		console.log('')
+		console.log('Transferring data between projects:')
+		console.log('')
+		console.log(`Source project name: ${sourceProject.project}`)
+		console.log(`Source API URL: ${sourceProject.baseUrl}`)
+		console.log(`Source token: ${maskToken(sourceProject.token)}`)
+		console.log('')
+		console.log(`Target project name: ${targetProject.project}`)
+		console.log(`Target API URL: ${targetProject.baseUrl}`)
+		console.log(`Target token: ${maskToken(targetProject.token)}`)
+		console.log('')
+		if (!await confirmImport(input)) {
+			return 1
+		}
+
+		const includeSystem = input.getOption('include-system') === true
+		const exportResponse = (await dataExport({ project: sourceProject, includeSystem, noGzip: true }))
+		const importResponse = await dataImport({
+			stream: exportResponse, project: targetProject, printProgress: rewriteStdoutLine,
+		})
+		console.log('')
+
+		const responseData = JSON.parse((await readStream(importResponse)).toString())
+		if (responseData.ok) {
+			console.log('Transfer done.')
+		} else {
+			console.error('Transfer failed:')
+			console.log(responseData)
+		}
+	}
+}

package/src/commands/transfer/index.ts

@@ -0,0 +1,3 @@
+export * from './ExportCommand'
+export * from './ImportCommand'
+export * from './TransferCommand'

package/src/commands/transfer/utils.ts

@@ -0,0 +1,135 @@
+import { Readable, Stream } from 'node:stream'
+import { LineTransform, readStream } from '../../utils/stream'
+import https from 'node:https'
+import http from 'node:http'
+import { Input, Workspace } from '@contember/cli-common'
+import { parseDsn } from '../../utils/dsn'
+import { interactiveResolveInstanceEnvironmentFromInput } from '../../utils/instance'
+import { interactiveResolveApiToken } from '../../utils/tenant'
+import prompts from 'prompts'
+
+export type ProjectEndpoint = {
+	token: string
+	project: string
+	baseUrl: string
+}
+
+export const resolveProject = async ({ projectOrDsn, workspace }: { projectOrDsn?: string; workspace: Workspace }): Promise<ProjectEndpoint> => {
+	let endpoint: string | undefined = undefined
+	let token: string | undefined = undefined
+	let project: string | undefined
+	if (projectOrDsn?.includes('://')) {
+		({ endpoint, project, token } = parseDsn(projectOrDsn))
+	} else if (projectOrDsn !== undefined && projectOrDsn !== '.') {
+		project = projectOrDsn
+	}
+	const instance = await interactiveResolveInstanceEnvironmentFromInput(workspace, endpoint)
+	token ??= await interactiveResolveApiToken({ workspace, instance })
+	project ??= (await workspace.projects.getSingleProject()).name
+	return { project, token, baseUrl: instance.baseUrl }
+
+}
+
+export const dataImport = async ({ printProgress, stream, project: { project, token, baseUrl } }: { stream: Readable; project: ProjectEndpoint; printProgress: (message: string) => void }) => {
+	let table = ''
+	let rowCount = 0
+	let rowTotal = 0
+	let transferred = 0
+	let start = Date.now()
+	const inputStream = stream.pipe(new LineTransform(line => {
+		transferred += line.length + 1
+		if (line === '') {
+			return ''
+		}
+		const data = JSON.parse(line)
+		switch (data[0]) {
+			case 'importSystemSchemaBegin':
+				return JSON.stringify([data[0], { ...data[1], project }])
+			case 'importContentSchemaBegin':
+				return JSON.stringify([data[0], { ...data[1], project }])
+			case 'insertBegin':
+				table = data[1].table
+				rowCount = 0
+				break
+			case 'insertRow':
+				rowCount++
+				rowTotal++
+				break
+			case 'insertEnd':
+				break
+		}
+		if ((rowCount % 1000) === 0 || data[0] !== 'insertRow') {
+			const mbs = Math.floor(transferred / 1024 / 1024).toString()
+			const durationS = Math.floor((Date.now() - start) / 1000)
+			printProgress(`transferred ${mbs} MiB; inserted ${rowCount} ${table} rows, ${rowTotal} total; ${durationS} seconds`)
+		}
+		return line
+	}))
+
+	return await executeRequest({
+		url: `${baseUrl}/import`,
+		token,
+		headers: {
+			'Content-type': 'application/x-ndjson',
+		},
+		body: inputStream,
+	})
+}
+
+export const dataExport = async ({ project: { project, token, baseUrl }, noGzip, includeSystem }: { project: ProjectEndpoint; includeSystem: boolean; noGzip: boolean }) => {
+	return await executeRequest({
+		url: `${baseUrl}/export`,
+		token,
+		headers: {
+			'Accept-Encoding': noGzip ? 'identity' : 'gzip',
+			'Content-type': 'application/json',
+		},
+		body: JSON.stringify({
+			projects: [{ slug: project, system: includeSystem }],
+		}),
+	})
+}
+
+const executeRequest = async ({ url, token, headers, body }: { url: string; token: string; headers: Record<string, string >; body: any }) => {
+	return await new Promise<http.IncomingMessage>((resolve, reject) => {
+		const req = (url.startsWith('https://') ? https : http).request(url, {
+			method: 'POST',
+			headers: {
+				Authorization: `Bearer ${token}`,
+				...headers,
+			},
+		}, async res => {
+			if (res.statusCode !== 200) {
+				const responseBody = (await readStream(res)).toString()
+				reject(new Error(`Invalid response: ${res.statusMessage}\n${responseBody}`))
+			} else {
+				resolve(res)
+			}
+		})
+
+		if (body instanceof Stream) {
+			body.pipe(req)
+		} else {
+			req.write(body)
+			req.end()
+		}
+	})
+}
+
+export const confirmImport = async (input: Input<{}, { yes: boolean }>): Promise<boolean> => {
+	if (input.getOption('yes')) {
+		return true
+	}
+	if (!process.stdin.isTTY) {
+		throw 'TTY not available. Pass --yes option to confirm execution.'
+	}
+	console.log('This will completely wipe the target project.')
+	console.log('(to skip this dialog, you can pass --yes option)')
+	console.log('')
+	const { ok } = await prompts({
+		type: 'confirm',
+		name: 'ok',
+		message: `Do you want to continue?`,
+	})
+	return ok
+}

package/src/run.ts

@@ -19,12 +19,18 @@ import {
 	VersionCommand,
 	WorkspaceUpdateApiCommand,
 } from './commands'
+import { ExportCommand } from './commands/transfer/ExportCommand'
+import { ImportCommand } from './commands/transfer/ImportCommand'
+import { TransferCommand } from './commands/transfer/TransferCommand';
 
 (async () => {
 	const workspace = await Workspace.get(process.cwd())
 	const commands: CommandFactoryList = {
 		['deploy']: () => new DeployCommand(workspace),
 		['version']: () => new VersionCommand(),
+		['data:export']: () => new ExportCommand(workspace),
+		['data:import']: () => new ImportCommand(workspace),
+		['data:transfer']: () => new TransferCommand(workspace),
 		['migrations:diff']: () => new MigrationDiffCommand(workspace),
 		['migrations:amend']: () => new MigrationAmendCommand(workspace),
 		['migrations:describe']: () => new MigrationDescribeCommand(workspace),

package/src/utils/admin.ts

@@ -1,5 +1,5 @@
 import fetch, { Response } from 'node-fetch'
-import { readdir, readFile } from 'fs/promises'
+import { readdir, readFile } from 'node:fs/promises'
 import { maskToken } from './token'
 
 export class AdminClient {

package/src/utils/commands.ts

@@ -1,5 +1,5 @@
-import { ChildProcessWithoutNullStreams, spawn } from 'child_process'
-import { Readable, Writable } from 'stream'
+import { ChildProcessWithoutNullStreams, spawn } from 'node:child_process'
+import { Readable, Writable } from 'node:stream'
 import chalk from 'chalk'
 
 export type RunningCommand = { child: ChildProcessWithoutNullStreams; output: Promise<string> }

package/src/utils/dockerCompose.ts

@@ -1,5 +1,5 @@
-import { join } from 'path'
-import { pathExists } from 'fs-extra'
+import { join } from 'node:path'
+import { pathExists } from '@contember/cli-common'
 import { JsonUpdateCallback, updateYaml } from './yaml'
 
 const MAIN_CONFIGS = ['docker-compose.yaml', 'docker-compose.yml']

package/src/utils/dsn.ts

@@ -0,0 +1,13 @@
+import { URL } from 'node:url'
+
+export const parseDsn = (dsn: string):  {endpoint: string; project: string; token: string} => {
+	const uri = new URL(dsn)
+	if (uri.protocol !== 'contember:' && uri.protocol !== 'contember-unsecure:') {
+		throw 'Invalid DSN'
+	}
+	return {
+		project: uri.username,
+		token: uri.password,
+		endpoint: (uri.protocol === 'contember-unsecure:' ? 'http://' : 'https://') + uri.host,
+	}
+}

package/src/utils/npm.ts

@@ -1,5 +1,5 @@
-import { pathExists } from 'fs-extra'
-import { join } from 'path'
+import { pathExists } from '@contember/cli-common'
+import { join } from 'node:path'
 import { runCommand } from './commands'
 import { listDirectories } from '@contember/cli-common'
 

package/src/utils/project/loadSchema.ts

@@ -1,7 +1,7 @@
 import { Project } from '@contember/cli-common'
 import { Schema } from '@contember/schema'
 import * as esbuild from 'esbuild'
-import { join } from 'path'
+import { join } from 'node:path'
 import { NodeVM } from 'vm2'
 import { schemaType } from '@contember/schema-utils'
 

package/src/utils/project/projectDescribe.tsx

@@ -2,7 +2,7 @@ import { Schema } from '@contember/schema'
 import { h } from 'preact'
 import render from 'preact-render-to-string'
 import { ProjectInfo } from './components/ProjectInfo'
-import { readFile } from 'fs/promises'
+import { readFile } from 'node:fs/promises'
 
 export const renderProjectInfoHtml = async (schema: Schema, projectName: string) => {
 	const html = render(<ProjectInfo schema={schema} projectName={projectName} />)

package/src/utils/schema.ts

@@ -1,4 +1,4 @@
-import { SchemaValidator, ValidationError } from '@contember/schema-utils'
+import { SchemaValidator, SchemaValidatorSkippedErrors, ValidationError } from '@contember/schema-utils'
 import { Schema } from '@contember/schema'
 
 export const printValidationErrors = (errors: ValidationError[], message?: string) => {
@@ -7,13 +7,13 @@ export const printValidationErrors = (errors: ValidationError[], message?: strin
 	}
 	console.group(message || 'Schema is invalid:')
 	for (const err of errors) {
-		console.log(err.path.join('.') + ': ' + err.message)
+		console.log(`${err.path.join('.')}: [${err.code}] ${err.message}`)
 	}
 	console.groupEnd()
 }
 
-export const validateSchemaAndPrintErrors = (schema: Schema, message?: string): boolean => {
-	const errors = SchemaValidator.validate(schema)
+export const validateSchemaAndPrintErrors = (schema: Schema, message?: string, skippedErrors?: SchemaValidatorSkippedErrors[]): boolean => {
+	const errors = SchemaValidator.validate(schema, skippedErrors)
 	printValidationErrors(errors, message)
 	return errors.length === 0
 }

package/src/utils/stdio.ts

@@ -0,0 +1,6 @@
+export const rewriteStdoutLine = (message: string) => {
+	process.stdout.clearLine(0)
+	process.stdout.cursorTo(0)
+	process.stdout.write(message)
+}
+

package/src/utils/stream.ts

@@ -0,0 +1,44 @@
+import { Stream, Transform } from 'node:stream'
+import { Buffer } from 'node:buffer'
+
+
+export class LineTransform extends Transform {
+	private chunks: Buffer[] = []
+
+	constructor(transform: (line: string) => string) {
+		super({
+			transform: (chunk, encoding, done) => {
+				while (true) {
+					const eolIndex = chunk.indexOf('\n')
+
+					if (eolIndex < 0) {
+						this.chunks.push(chunk)
+						break
+					}
+
+					this.chunks.push(chunk.slice(0, eolIndex))
+					chunk = chunk.slice(eolIndex + 1)
+					this.push(transform(Buffer.concat(this.chunks).toString('utf8')) + '\n', 'utf8')
+					this.chunks = []
+				}
+
+				return done()
+			},
+
+			flush: done => {
+				if (this.chunks.length) {
+					this.push(transform(Buffer.concat(this.chunks).toString('utf8')), 'utf8')
+					return done()
+				}
+			},
+		})
+	}
+}
+
+
+export const readStream = async (stream: Stream) => new Promise<Buffer>((resolve, reject) => {
+	const chunks: Buffer[] = []
+	stream.on('error', err => reject(err))
+	stream.on('data', chunk => chunks.push(chunk))
+	stream.on('end', () => resolve(Buffer.concat(chunks)))
+})

package/src/utils/workspace.ts

@@ -1,5 +1,4 @@
-import { pathExists } from 'fs-extra'
-import { formatWorkspaceConfigPath, Workspace } from '@contember/cli-common'
+import { formatWorkspaceConfigPath, pathExists, Workspace } from '@contember/cli-common'
 import { updateYaml } from './yaml'
 
 export const updateWorkspaceApiVersion = async (workspace: Workspace, newVersion: string): Promise<string | null> => {

package/src/utils/yaml.ts

@@ -1,9 +1,8 @@
-import { JSONObject, JSONPrimitive } from '@contember/cli-common/dist/src/utils/json'
-import { pathExists } from 'fs-extra'
+import { JSONObject, JSONPrimitive, pathExists } from '@contember/cli-common'
 import YAWN from 'yawn-yaml/cjs'
 import { Merger } from '@contember/config-loader'
 import jsyaml from 'js-yaml'
-import { readFile, writeFile } from 'fs/promises'
+import { readFile, writeFile } from 'node:fs/promises'
 
 export type JsonUpdateCallback<Value = JSONObject> = (
 	data: Partial<Value>,