@lincyaw/workspace 1.0.0-rc.0 → 1.0.0-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -5
- package/package.json +10 -5
- package/src/domain/{revisions.ts → commit-message.ts} +7 -7
- package/src/domain/locks.ts +5 -0
- package/src/domain/paths.ts +2 -2
- package/src/index.ts +7 -2
- package/src/node/authoring-clients.ts +206 -0
- package/src/node/build.ts +4 -4
- package/src/node/compile-session.ts +187 -0
- package/src/node/config.ts +36 -21
- package/src/node/document-assets.ts +30 -0
- package/src/node/document-audio.ts +473 -0
- package/src/node/document-authoring.ts +436 -0
- package/src/node/document-editor.ts +157 -0
- package/src/node/document-history.ts +190 -0
- package/src/node/document-images.ts +241 -0
- package/src/node/document-operations.ts +191 -0
- package/src/node/document-transcription-adoption.ts +96 -0
- package/src/node/document-transcription.ts +471 -0
- package/src/node/document-voice.ts +115 -0
- package/src/node/errors.ts +23 -0
- package/src/node/git.ts +15 -15
- package/src/node/jobs.ts +361 -0
- package/src/node/locks.ts +10 -5
- package/src/node/page-delete.ts +106 -0
- package/src/node/project-files.ts +163 -0
- package/src/node/projects.ts +192 -0
- package/src/node/review/agent-session-reviews.ts +374 -0
- package/src/node/review/git-guard.ts +9 -8
- package/src/node/review/git-review.ts +102 -90
- package/src/node/review/record.ts +77 -0
- package/src/node/review/routes.ts +36 -35
- package/src/node/review/task-reviews.ts +50 -0
- package/src/node/revisions.ts +321 -68
- package/src/node/routes.ts +236 -146
- package/src/node/serve.ts +319 -118
- package/src/node/site.ts +34 -23
- package/src/node/source-editor.ts +332 -0
- package/src/node/speech-settings.ts +21 -0
- package/src/node/templates.ts +27 -0
- package/src/node/workbench-api.ts +176 -0
- package/src/node/writer.ts +389 -0
- package/templates/document/default/lecture.json +75 -0
- package/src/domain/api.ts +0 -102
- package/src/domain/index.ts +0 -6
- package/src/node/documents.ts +0 -185
- package/src/node/files.ts +0 -252
- package/src/node/review/review-record.ts +0 -73
- package/src/node/review/reviews.ts +0 -422
- package/src/node/service.ts +0 -237
- package/src/node/session.ts +0 -123
- package/src/node/source.ts +0 -148
- package/templates/document/default/lecture.md +0 -18
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { DOCUMENT_ENTRY, fnv1a64, hasRole, speechEngineId } from '@lincyaw/kernel'
|
|
2
|
+
import type { CanonicalSource, DocumentVoiceAuthoring, Finding, Member, SpeechService, VoiceoverExecution, VoiceoverJob } from '@lincyaw/kernel'
|
|
3
|
+
import { createVoiceoverOwner, createVoiceoverRoutes, validateDocumentVoiceCatalogQuery, validateVoicePreviewStart } from '@lincyaw/kernel/node'
|
|
4
|
+
import { readAuthoringDocument } from './document-editor.ts'
|
|
5
|
+
import { RequestError } from './errors.ts'
|
|
6
|
+
import type { SourceEditor } from './source-editor.ts'
|
|
7
|
+
|
|
8
|
+
export interface DocumentVoiceServices { speech?: SpeechService, voiceover?: { version: 1, preview: true }, voiceoverExecution?: VoiceoverExecution, findings?: Finding[] }
|
|
9
|
+
export interface DocumentVoiceAuthoringOptions {
|
|
10
|
+
editor: SourceEditor
|
|
11
|
+
services(document: string, source: CanonicalSource): Promise<DocumentVoiceServices>
|
|
12
|
+
revision?(document: string): string | undefined
|
|
13
|
+
}
|
|
14
|
+
const same = (a: CanonicalSource, b: CanonicalSource) => a.documentId === b.documentId && a.version === b.version && a.fingerprint === b.fingerprint
|
|
15
|
+
|
|
16
|
+
export function createDocumentVoiceAuthoring({ editor, services, revision }: DocumentVoiceAuthoringOptions) {
|
|
17
|
+
const pending = new Set<{ controller: AbortController, finished: Promise<unknown> }>()
|
|
18
|
+
const owners = new Map<string, ReturnType<typeof createVoiceoverOwner>>()
|
|
19
|
+
let closed = false
|
|
20
|
+
const authorize = (document: string, member: Member) => {
|
|
21
|
+
if (!hasRole(member, 'editor')) throw new RequestError(403, 'voice-authoring-permission-denied')
|
|
22
|
+
if (closed) throw new RequestError(503, 'voice-authoring-closed')
|
|
23
|
+
if (!owners.has(document)) editor.readableFile({ document, home: false }, document ? `${document}/${DOCUMENT_ENTRY}` : DOCUMENT_ENTRY, member)
|
|
24
|
+
}
|
|
25
|
+
const ownerOf = (document: string, member: Member) => {
|
|
26
|
+
authorize(document, member)
|
|
27
|
+
let owner = owners.get(document)
|
|
28
|
+
if (!owner) { owner = createVoiceoverOwner(); owners.set(document, owner) }
|
|
29
|
+
return owner
|
|
30
|
+
}
|
|
31
|
+
const current = (document: string, member: Member, expected?: CanonicalSource) => {
|
|
32
|
+
const opened = readAuthoringDocument(editor, { document, member })
|
|
33
|
+
const source = { documentId: opened.content.id, version: opened.source.version, fingerprint: fnv1a64(opened.source.content) }
|
|
34
|
+
if (expected && !same(source, expected)) throw new RequestError(409, 'document-version-changed', { currentSource: source })
|
|
35
|
+
return { source, lang: opened.content.metadata.lang }
|
|
36
|
+
}
|
|
37
|
+
const work = <T>(signal: AbortSignal | undefined, run: (signal: AbortSignal) => Promise<T>): Promise<T> => {
|
|
38
|
+
if (closed) throw new RequestError(503, 'voice-authoring-closed')
|
|
39
|
+
signal?.throwIfAborted()
|
|
40
|
+
if (pending.size >= 4) throw new RequestError(503, 'voice-authoring-busy')
|
|
41
|
+
const controller = new AbortController(), combined = signal ? AbortSignal.any([signal, controller.signal]) : controller.signal
|
|
42
|
+
const operation = { controller, finished: Promise.resolve() as Promise<unknown> }
|
|
43
|
+
pending.add(operation)
|
|
44
|
+
const finished = Promise.resolve().then(() => { combined.throwIfAborted(); return run(combined) }).finally(() => pending.delete(operation))
|
|
45
|
+
operation.finished = finished
|
|
46
|
+
return finished
|
|
47
|
+
}
|
|
48
|
+
const load = async (document: string, member: Member, signal: AbortSignal, expected?: CanonicalSource) => {
|
|
49
|
+
authorize(document, member)
|
|
50
|
+
const opened = current(document, member, expected), loaded = await services(document, opened.source)
|
|
51
|
+
signal.throwIfAborted(); current(document, member, opened.source)
|
|
52
|
+
return { ...opened, ...loaded }
|
|
53
|
+
}
|
|
54
|
+
const execution = (document: string, member: Member) => async (expected: CanonicalSource | undefined, signal: AbortSignal) => {
|
|
55
|
+
const loaded = await load(document, member, signal, expected)
|
|
56
|
+
if (!loaded.voiceoverExecution) throw new RequestError(503, 'voiceover-unavailable')
|
|
57
|
+
return loaded.voiceoverExecution
|
|
58
|
+
}
|
|
59
|
+
const isCurrent = (document: string, member: Member, job: VoiceoverJob): boolean | 'unknown' => {
|
|
60
|
+
try { const actual = current(document, member).source; if (!same(actual, job.source)) return false; const known = revision?.(document); return known === undefined ? 'unknown' : known === job.revision }
|
|
61
|
+
catch { return 'unknown' }
|
|
62
|
+
}
|
|
63
|
+
const service: DocumentVoiceAuthoring = {
|
|
64
|
+
capabilities: (document, member, signal) => work(signal, async signal => {
|
|
65
|
+
const loaded = await load(document, member, signal)
|
|
66
|
+
const [engines, defaultEngine] = await Promise.all([loaded.speech?.engines() ?? [], loaded.speech?.defaultSynthesisEngine?.()])
|
|
67
|
+
signal.throwIfAborted(); current(document, member, loaded.source)
|
|
68
|
+
const configured = engines.some(engine => engine.tts), unknown = !loaded.speech && loaded.findings?.some(finding => finding.severity === 'error')
|
|
69
|
+
const implemented = Boolean(loaded.voiceoverExecution)
|
|
70
|
+
return { document, source: loaded.source, verification: 'configuration-only', status: configured ? 'configured' : unknown ? 'unknown' : 'unavailable', ...!configured ? { reason: unknown ? 'services-unavailable' : 'not-configured' } as const : {}, voiceover: implemented, preview: implemented, engines, ...defaultEngine ? { defaultEngine } : {}, ...loaded.findings?.length ? { findings: loaded.findings } : {}, actions: { status: implemented, list: true, get: true, start: implemented && configured, cancel: true, quotePreview: implemented, preview: implemented && configured, readPreview: true }, tasks: { idempotency: 'process', limits: ownerOf(document, member).limits } }
|
|
71
|
+
}),
|
|
72
|
+
catalog: (document, query, member, signal) => work(signal, async signal => {
|
|
73
|
+
if (!validateDocumentVoiceCatalogQuery(query)) throw new RequestError(400, 'voice-catalog-query-invalid')
|
|
74
|
+
const loaded = await load(document, member, signal, query.source)
|
|
75
|
+
const engines = await loaded.speech?.engines() ?? []
|
|
76
|
+
signal.throwIfAborted(); current(document, member, loaded.source)
|
|
77
|
+
const preferred = query.engine ?? await loaded.speech?.defaultSynthesisEngine?.()
|
|
78
|
+
signal.throwIfAborted(); current(document, member, loaded.source)
|
|
79
|
+
const chosen = preferred ? engines.find(engine => engine.tts && engine.id === speechEngineId(preferred)) : undefined
|
|
80
|
+
const unknown = !loaded.speech && loaded.findings?.some(finding => finding.severity === 'error')
|
|
81
|
+
const catalog = chosen && loaded.speech ? await loaded.speech.voiceCatalog?.(chosen.id, query.lang ?? loaded.lang, signal) ?? { status: 'unavailable' as const, reason: 'unknown' as const, voices: [] } : { status: 'unavailable' as const, reason: unknown ? 'unknown' as const : 'not-configured' as const, voices: [] }
|
|
82
|
+
signal.throwIfAborted(); current(document, member, loaded.source)
|
|
83
|
+
return { document, source: loaded.source, engines, catalog, ...catalog.status === 'available' && chosen ? { engine: chosen.id } : {}, ...loaded.findings?.length ? { findings: loaded.findings } : {} }
|
|
84
|
+
}),
|
|
85
|
+
snapshot: (document, query, member, signal) => ownerOf(document, member).snapshot(query, execution(document, member), signal),
|
|
86
|
+
listJobs: async (document, member) => {
|
|
87
|
+
const owner = ownerOf(document, member)
|
|
88
|
+
return { document, jobs: owner.list().map(job => ({ ...job, current: isCurrent(document, member, job) })), idempotency: 'process', limits: owner.limits }
|
|
89
|
+
},
|
|
90
|
+
getJob: async (document, query, member) => {
|
|
91
|
+
const value = ownerOf(document, member).get(query)
|
|
92
|
+
return { ...value, current: value.job ? isCurrent(document, member, value.job) : 'unknown' }
|
|
93
|
+
},
|
|
94
|
+
start: (document, request, member, signal) => ownerOf(document, member).start(request, execution(document, member), signal),
|
|
95
|
+
checkStart: (document, request, member, signal) => ownerOf(document, member).checkStart(request, execution(document, member), signal),
|
|
96
|
+
checkCancel: async (document, query, member) => ownerOf(document, member).checkCancel(query),
|
|
97
|
+
checkPreview: async (document, request, member, signal) => {
|
|
98
|
+
if (!validateVoicePreviewStart(request)) throw new RequestError(400, 'voice-preview-invalid')
|
|
99
|
+
const owner = ownerOf(document, member)
|
|
100
|
+
current(document, member, request.source)
|
|
101
|
+
await owner.checkPreview(request, signal)
|
|
102
|
+
current(document, member, request.source)
|
|
103
|
+
},
|
|
104
|
+
cancel: async (document, query, member) => ownerOf(document, member).cancel(query),
|
|
105
|
+
quotePreview: (document, query, member, signal) => ownerOf(document, member).quote(query, execution(document, member), signal),
|
|
106
|
+
preview: (document, request, member, signal) => {
|
|
107
|
+
if (!validateVoicePreviewStart(request)) throw new RequestError(400, 'voice-preview-invalid')
|
|
108
|
+
const owner = ownerOf(document, member)
|
|
109
|
+
current(document, member, request.source)
|
|
110
|
+
return owner.preview(request, signal)
|
|
111
|
+
},
|
|
112
|
+
readPreview: async (document, id, member) => ownerOf(document, member).readPreview(id),
|
|
113
|
+
}
|
|
114
|
+
return { service, routes: createVoiceoverRoutes(service), async close() { closed = true; for (const operation of pending) operation.controller.abort(); await Promise.allSettled([...owners.values()].map(owner => owner.close())); await Promise.allSettled([...pending].map(operation => operation.finished)); owners.clear() } }
|
|
115
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { errorMessage, jsonResponse, problemResponse } from '@lincyaw/kernel'
|
|
2
|
+
|
|
3
|
+
export class RequestError extends Error {
|
|
4
|
+
readonly status: number
|
|
5
|
+
readonly detail?: Record<string, unknown>
|
|
6
|
+
readonly headers?: Record<string, string>
|
|
7
|
+
constructor(status: number, message: string, detail?: Record<string, unknown>, headers?: Record<string, string>) {
|
|
8
|
+
super(message)
|
|
9
|
+
this.status = status
|
|
10
|
+
this.detail = detail
|
|
11
|
+
this.headers = headers
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/** Runs a route's work and answers a RequestError it throws with its status, detail and headers, anything else with 500. */
|
|
16
|
+
export async function catchRequestErrors(work: () => Promise<Response>): Promise<Response> {
|
|
17
|
+
try {
|
|
18
|
+
return await work()
|
|
19
|
+
} catch (error) {
|
|
20
|
+
if (error instanceof RequestError) return jsonResponse({ error: error.message, ...error.detail }, error.status, error.headers)
|
|
21
|
+
return problemResponse(500, errorMessage(error))
|
|
22
|
+
}
|
|
23
|
+
}
|
package/src/node/git.ts
CHANGED
|
@@ -26,36 +26,36 @@ export class Git {
|
|
|
26
26
|
}
|
|
27
27
|
}
|
|
28
28
|
|
|
29
|
-
|
|
30
|
-
try {
|
|
31
|
-
execFileSync('git', [...gitConfigArgs(), 'check-ignore', '-q', '--', path], { cwd: dir, stdio: 'ignore', env: gitEnvironment(process.env) })
|
|
32
|
-
return true
|
|
33
|
-
} catch {
|
|
34
|
-
return false
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
run(command: string[], env: Record<string, string> = {}): Promise<GitResult> {
|
|
29
|
+
run(command: string[], env: Record<string, string> = {}, input?: Uint8Array): Promise<GitResult> {
|
|
39
30
|
return new Promise((done) => {
|
|
40
|
-
|
|
31
|
+
let inputError: Error | undefined
|
|
32
|
+
const child = execFile('git', [...gitConfigArgs(), ...command], {
|
|
41
33
|
cwd: this.root,
|
|
42
34
|
encoding: 'buffer',
|
|
43
35
|
maxBuffer: 1 << 30,
|
|
44
36
|
env: { ...gitEnvironment(process.env), ...env },
|
|
45
37
|
}, (error, stdout, stderr) => {
|
|
46
|
-
const code = !error ? 0 : typeof error.code === 'number' ? error.code : 1
|
|
47
|
-
done({ code, stdout, stderr: stderr.toString('utf8') })
|
|
38
|
+
const code = inputError ? 1 : !error ? 0 : typeof error.code === 'number' ? error.code : 1
|
|
39
|
+
done({ code, stdout, stderr: stderr.toString('utf8') + (inputError?.message ?? '') })
|
|
48
40
|
})
|
|
41
|
+
if (input !== undefined) {
|
|
42
|
+
if (child.stdin) {
|
|
43
|
+
child.stdin.on('error', error => { inputError = error; child.kill() })
|
|
44
|
+
child.stdin.end(input)
|
|
45
|
+
} else { inputError = new Error('git stdin is unavailable'); child.kill() }
|
|
46
|
+
}
|
|
49
47
|
})
|
|
50
48
|
}
|
|
51
49
|
|
|
52
|
-
|
|
50
|
+
/** The command's output; throws when it fails. */
|
|
51
|
+
async output(command: string[], env?: Record<string, string>): Promise<string> {
|
|
53
52
|
const result = await this.run(command, env)
|
|
54
53
|
if (result.code !== 0) throw new Error(result.stderr.trim() || `git ${command[0]} exited with ${result.code}`)
|
|
55
54
|
return result.stdout.toString('utf8')
|
|
56
55
|
}
|
|
57
56
|
|
|
58
|
-
|
|
57
|
+
/** Whether the command succeeded, with its output, or its error output when it failed. */
|
|
58
|
+
async tryOutput(command: string[], env?: Record<string, string>): Promise<{ ok: boolean, output: string }> {
|
|
59
59
|
const result = await this.run(command, env)
|
|
60
60
|
return { ok: result.code === 0, output: result.code === 0 ? result.stdout.toString('utf8') : `${result.stderr}${result.stdout.toString('utf8')}` }
|
|
61
61
|
}
|
package/src/node/jobs.ts
ADDED
|
@@ -0,0 +1,361 @@
|
|
|
1
|
+
import { createHash, randomBytes } from 'node:crypto'
|
|
2
|
+
import { createReadStream, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'
|
|
3
|
+
import { join, relative, sep } from 'node:path'
|
|
4
|
+
import { availableRenderers, build, compileTranslations, documentExporters, exportDocument, serveDirectory } from '@lincyaw/compiler'
|
|
5
|
+
import type { Compilation } from '@lincyaw/compiler'
|
|
6
|
+
import { errorMessage, fnv1a64 } from '@lincyaw/kernel'
|
|
7
|
+
import { lookup } from 'mime-types'
|
|
8
|
+
import type { CompileJob, JobFile, JobSource, Logger, OutputJobs, OutputTarget } from '@lincyaw/kernel'
|
|
9
|
+
import { RequestError } from './errors.ts'
|
|
10
|
+
import { versionOf } from './source-editor.ts'
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* The documents' heavy outputs as jobs, one running at a time: the renderers' builds (the slides' site, the notebook, the
|
|
14
|
+
* lecture) and the plugins' exporters (the slides as PDF or PowerPoint, the lecture as a video). A job shows its progress
|
|
15
|
+
* and log; its files are kept below the service's `/__outputs/<job>/`, and a job of the same target and IR revision is
|
|
16
|
+
* not run again.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
export const OUTPUTS_PATH = '/__outputs/'
|
|
20
|
+
const JOB_FILE = 'job.json'
|
|
21
|
+
// The last lines of a job's log the job keeps.
|
|
22
|
+
const LOG_LINES = 400
|
|
23
|
+
// The jobs a document keeps of each target, with their files.
|
|
24
|
+
const KEEP = 3
|
|
25
|
+
// The files a job lists (a site has many; the rest are still served).
|
|
26
|
+
const LISTED = 500
|
|
27
|
+
// Progress and log lines are told at most this often.
|
|
28
|
+
const TELL_MS = 250
|
|
29
|
+
|
|
30
|
+
export interface JobsOptions {
|
|
31
|
+
stateDir: string
|
|
32
|
+
/** The service's origin as browsers use it. */
|
|
33
|
+
url(): string
|
|
34
|
+
/** The path of the jobs' files on that origin (default `/__outputs/`; below the base when a proxy fronts the service). */
|
|
35
|
+
outputsPath?: string
|
|
36
|
+
compilation(document: string, fresh?: boolean): Promise<Compilation>
|
|
37
|
+
revision(compilation: Compilation): string
|
|
38
|
+
/** The workbench's origins, whose pages may read the files (a video's captions). */
|
|
39
|
+
origins: string[]
|
|
40
|
+
log: Logger
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export interface Jobs extends OutputJobs {
|
|
44
|
+
get(id: string): CompileJob | undefined
|
|
45
|
+
/** Where a job's output is on this machine: a renderer's site or files, an exporter's files. */
|
|
46
|
+
directory(job: CompileJob): string
|
|
47
|
+
/** Each change of a job (state, progress, log), as it happens. */
|
|
48
|
+
subscribe(listener: (job: CompileJob) => void): () => void
|
|
49
|
+
/** The address that serves the jobs' files (the service forwards `/__outputs/` to it). */
|
|
50
|
+
origin(): Promise<string>
|
|
51
|
+
close(): Promise<void>
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
interface Running {
|
|
55
|
+
job: CompileJob
|
|
56
|
+
dir: string
|
|
57
|
+
controller: AbortController
|
|
58
|
+
ended: Promise<CompileJob>
|
|
59
|
+
end(): void
|
|
60
|
+
told: number
|
|
61
|
+
timer?: ReturnType<typeof setTimeout>
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
class Cancelled extends Error {
|
|
65
|
+
constructor() {
|
|
66
|
+
super('the job was cancelled')
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// Where an exporter's renderer project is written, inside the job's directory, removed when it ends.
|
|
71
|
+
|
|
72
|
+
const now = () => new Date().toISOString()
|
|
73
|
+
const newId = () => `${Date.now().toString(36)}${randomBytes(4).toString('hex')}`
|
|
74
|
+
const sameSource = (a: JobSource | undefined, b: JobSource): boolean => a?.version === b.version && a.fingerprint === b.fingerprint
|
|
75
|
+
const sourceOf = (compilation: Compilation): JobSource => {
|
|
76
|
+
const bytes = readFileSync(compilation.entry)
|
|
77
|
+
return { version: versionOf(bytes), fingerprint: fnv1a64(bytes.toString('utf8')) }
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** Files a renderer/exporter may read after compilation must remain the captured inputs until output completion. */
|
|
81
|
+
async function inputVersion(file: string): Promise<string | undefined> {
|
|
82
|
+
try {
|
|
83
|
+
const hash = createHash('sha256')
|
|
84
|
+
// Media inputs can be large. Hash streams without blocking requests or retaining the whole file in memory.
|
|
85
|
+
for await (const chunk of createReadStream(file)) hash.update(chunk)
|
|
86
|
+
return hash.digest('hex')
|
|
87
|
+
} catch { return undefined }
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
async function inputVersions(files: Iterable<string>): Promise<Map<string, string | undefined>> {
|
|
91
|
+
const pending = [...new Set(files)], versions = new Map<string, string | undefined>()
|
|
92
|
+
await Promise.all(Array.from({ length: Math.min(4, pending.length) }, async () => {
|
|
93
|
+
let file: string | undefined
|
|
94
|
+
while ((file = pending.pop()) !== undefined) versions.set(file, await inputVersion(file))
|
|
95
|
+
}))
|
|
96
|
+
return versions
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
async function checkInputs(inputs: Map<string, string | undefined>): Promise<void> {
|
|
100
|
+
const actual = await inputVersions(inputs.keys())
|
|
101
|
+
for (const [file, expected] of inputs) if (actual.get(file) !== expected) throw new RequestError(409, 'job-inputs-changed')
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** A file of a job, its path relative to the job's directory, its media type by its extension (`mime-types`). */
|
|
105
|
+
function jobFile(root: string, path: string): JobFile {
|
|
106
|
+
return { path: relative(root, path).split(sep).join('/'), size: statSync(path).size, type: lookup(path) || 'application/octet-stream' }
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** The files below `dir`, at most LISTED, paths relative to `root`. */
|
|
110
|
+
function listFiles(root: string, dir: string): JobFile[] {
|
|
111
|
+
const files: JobFile[] = []
|
|
112
|
+
const walk = (at: string) => {
|
|
113
|
+
for (const entry of readdirSync(at, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
|
|
114
|
+
if (files.length >= LISTED) return
|
|
115
|
+
const path = join(at, entry.name)
|
|
116
|
+
if (entry.isDirectory()) walk(path)
|
|
117
|
+
else if (entry.isFile() && entry.name !== JOB_FILE) files.push(jobFile(root, path))
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
if (existsSync(dir)) walk(dir)
|
|
121
|
+
return files
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/** The file of a renderer's output to open first: a site's page, else its first file. */
|
|
125
|
+
function mainFile(target: string, files: JobFile[]): string | undefined {
|
|
126
|
+
return files.find(file => file.path === `${target}/index.html`)?.path ?? files.find(file => file.path.startsWith(`${target}/`))?.path
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
export function createJobs(options: JobsOptions): Jobs {
|
|
130
|
+
const outputs = options.outputsPath ?? OUTPUTS_PATH
|
|
131
|
+
const root = join(options.stateDir, 'outputs')
|
|
132
|
+
// The jobs, oldest first, without their address (`present` adds it: the service's port changes between sessions).
|
|
133
|
+
const jobs = new Map<string, CompileJob>()
|
|
134
|
+
const running = new Map<string, Running>()
|
|
135
|
+
const listeners = new Set<(job: CompileJob) => void>()
|
|
136
|
+
let queue = Promise.resolve()
|
|
137
|
+
let server: ReturnType<typeof serveDirectory> | undefined
|
|
138
|
+
let closed = false
|
|
139
|
+
|
|
140
|
+
const present = (job: CompileJob): CompileJob => ({ ...job, url: `${options.url()}${outputs}${job.id}/`, log: [...job.log], files: [...job.files] })
|
|
141
|
+
|
|
142
|
+
// The jobs of earlier sessions whose files are still there.
|
|
143
|
+
mkdirSync(root, { recursive: true })
|
|
144
|
+
for (const id of readdirSync(root).sort()) {
|
|
145
|
+
try {
|
|
146
|
+
const job = JSON.parse(readFileSync(join(root, id, JOB_FILE), 'utf8')) as CompileJob
|
|
147
|
+
if (job.state === 'done' && job.id === id) jobs.set(id, { ...job, sequence: job.sequence ?? 0 })
|
|
148
|
+
else rmSync(join(root, id), { recursive: true, force: true })
|
|
149
|
+
} catch {
|
|
150
|
+
rmSync(join(root, id), { recursive: true, force: true })
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
const tell = (job: CompileJob) => {
|
|
155
|
+
job.sequence++
|
|
156
|
+
const shown = present(job)
|
|
157
|
+
for (const listener of listeners) listener(shown)
|
|
158
|
+
}
|
|
159
|
+
/** Tells a change at once (state), or within TELL_MS (progress, log). */
|
|
160
|
+
const changed = (entry: Running, soon = false) => {
|
|
161
|
+
clearTimeout(entry.timer)
|
|
162
|
+
entry.timer = setTimeout(() => {
|
|
163
|
+
entry.told = Date.now()
|
|
164
|
+
tell(entry.job)
|
|
165
|
+
}, soon ? 0 : Math.max(0, entry.told + TELL_MS - Date.now()))
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/** Leaves the newest KEEP ended jobs of the document's target, with their files. */
|
|
169
|
+
const prune = (document: string, target: string) => {
|
|
170
|
+
const ended = [...jobs.values()].filter(job => job.document === document && job.target === target && !running.has(job.id))
|
|
171
|
+
for (const job of ended.slice(0, Math.max(0, ended.length - KEEP))) {
|
|
172
|
+
jobs.delete(job.id)
|
|
173
|
+
rmSync(join(root, job.id), { recursive: true, force: true })
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
const targetsOf = async (compilation: Compilation): Promise<OutputTarget[]> => {
|
|
178
|
+
const renderers = await availableRenderers(compilation)
|
|
179
|
+
const exports = (await documentExporters(compilation))
|
|
180
|
+
.filter(({ exporter, config }) => !renderers.includes(exporter.id) && (exporter.available?.(compilation.doc, config) ?? true))
|
|
181
|
+
.map(({ exporter }): OutputTarget => ({ id: exporter.id, kind: 'export', label: exporter.label }))
|
|
182
|
+
return [...renderers.map((id): OutputTarget => ({ id, kind: 'render', label: { 'zh-CN': id, 'en': id } })), ...exports]
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
const run = async (entry: Running, compilation: Compilation) => {
|
|
186
|
+
const { job, dir, controller } = entry
|
|
187
|
+
const log = (line: string) => {
|
|
188
|
+
options.log.info(`[job ${job.id}] document=${JSON.stringify(job.document || '.')} target=${job.target} ${line}`)
|
|
189
|
+
job.log.push(line)
|
|
190
|
+
if (job.log.length > LOG_LINES) job.log.splice(0, job.log.length - LOG_LINES)
|
|
191
|
+
changed(entry)
|
|
192
|
+
}
|
|
193
|
+
const progress = (phase: string, done?: number, total?: number) => {
|
|
194
|
+
if (job.progress?.phase !== phase) options.log.info(`[job ${job.id}] document=${JSON.stringify(job.document || '.')} target=${job.target} phase=${JSON.stringify(phase)}`)
|
|
195
|
+
job.progress = { phase, ...done === undefined ? {} : { done }, ...total === undefined ? {} : { total } }
|
|
196
|
+
changed(entry)
|
|
197
|
+
}
|
|
198
|
+
if (job.kind === 'render') {
|
|
199
|
+
progress(job.target)
|
|
200
|
+
const report = await build(compilation, { outDir: dir, to: [job.target], base: `${outputs}${job.id}/${job.target}/`, log })
|
|
201
|
+
if (controller.signal.aborted) throw new Cancelled()
|
|
202
|
+
job.warnings = report.findings.filter(finding => finding.severity === 'warning').map(finding => finding.message)
|
|
203
|
+
if (!report.outputs.some(output => output.renderer === job.target)) {
|
|
204
|
+
throw new Error(report.findings.filter(finding => finding.severity === 'error').map(finding => finding.message).join('\n') || `the ${job.target} renderer made nothing`)
|
|
205
|
+
}
|
|
206
|
+
job.files = listFiles(dir, join(dir, job.target))
|
|
207
|
+
job.main = mainFile(job.target, job.files)
|
|
208
|
+
return
|
|
209
|
+
}
|
|
210
|
+
const output = await exportDocument(compilation, job.target, { outDir: dir, signal: controller.signal, progress, log })
|
|
211
|
+
if (controller.signal.aborted) throw new Cancelled()
|
|
212
|
+
job.warnings = output.warnings ?? []
|
|
213
|
+
const named = output.files.map(file => file.split(sep).join('/'))
|
|
214
|
+
const first = named.flatMap(path => (existsSync(join(dir, path)) ? [jobFile(dir, join(dir, path))] : []))
|
|
215
|
+
job.files = [...first, ...listFiles(dir, dir).filter(file => !named.includes(file.path))]
|
|
216
|
+
job.main = first[0]?.path
|
|
217
|
+
if (output.captions) job.captions = Object.fromEntries(Object.entries(output.captions).map(([lang, file]) => [lang, file.split(sep).join('/')]))
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/** Runs the job once the ones before it have ended. */
|
|
221
|
+
const schedule = (entry: Running, compilation: Compilation, inputs: Map<string, string | undefined>) => {
|
|
222
|
+
queue = queue.then(async () => {
|
|
223
|
+
const { job, controller } = entry
|
|
224
|
+
if (job.state !== 'queued') return
|
|
225
|
+
job.state = 'running'
|
|
226
|
+
job.startedAt = now()
|
|
227
|
+
const started = Date.now()
|
|
228
|
+
options.log.info(`[job ${job.id}] document=${JSON.stringify(job.document || '.')} target=${job.target} state=running queuedMs=${started - Date.parse(job.createdAt)}`)
|
|
229
|
+
changed(entry, true)
|
|
230
|
+
try {
|
|
231
|
+
await checkInputs(inputs)
|
|
232
|
+
if (controller.signal.aborted) throw new Cancelled()
|
|
233
|
+
mkdirSync(entry.dir, { recursive: true })
|
|
234
|
+
await run(entry, compilation)
|
|
235
|
+
await checkInputs(inputs)
|
|
236
|
+
if (controller.signal.aborted) throw new Cancelled()
|
|
237
|
+
job.state = 'done'
|
|
238
|
+
} catch (error) {
|
|
239
|
+
job.state = controller.signal.aborted ? 'cancelled' : 'failed'
|
|
240
|
+
if (job.state === 'failed') job.error = errorMessage(error)
|
|
241
|
+
options.log.warn(`the ${job.target} of ${job.document || '.'} ${job.state === 'cancelled' ? 'was cancelled' : `failed: ${job.error}`}`)
|
|
242
|
+
}
|
|
243
|
+
job.finishedAt = now()
|
|
244
|
+
options.log.info(`[job ${job.id}] document=${JSON.stringify(job.document || '.')} target=${job.target} state=${job.state} elapsedMs=${Date.now() - started}`)
|
|
245
|
+
delete job.progress
|
|
246
|
+
if (job.state === 'done') writeFileSync(join(entry.dir, JOB_FILE), `${JSON.stringify(job, null, 2)}\n`)
|
|
247
|
+
else rmSync(entry.dir, { recursive: true, force: true })
|
|
248
|
+
entry.end()
|
|
249
|
+
})
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
/** Everything start checks before it queues a job, and the job of the same target, source and inputs it would reuse. */
|
|
253
|
+
const admission = async (document: string, target: string, expected?: JobSource) => {
|
|
254
|
+
if (closed) throw new Error('the workspace service is closing')
|
|
255
|
+
// The interactive preview may still hold an older successful compilation. An explicit output compiles afresh.
|
|
256
|
+
const compilation = await options.compilation(document, true)
|
|
257
|
+
const source = sourceOf(compilation)
|
|
258
|
+
const confirmed = () => {
|
|
259
|
+
if (closed) throw new Error('the workspace service is closing')
|
|
260
|
+
if (expected && !sameSource(expected, source) || !sameSource(sourceOf(compilation), source)) throw new RequestError(409, 'job-source-changed')
|
|
261
|
+
if (compilation.sourceFingerprints?.[compilation.entry] !== source.fingerprint) throw new RequestError(409, 'job-source-not-compiled')
|
|
262
|
+
}
|
|
263
|
+
confirmed()
|
|
264
|
+
const revision = options.revision(compilation)
|
|
265
|
+
const targets = await targetsOf(compilation)
|
|
266
|
+
const known = targets.find(own => own.id === target)
|
|
267
|
+
if (!known) throw new RequestError(422, 'job-target-unknown', { target, targets: targets.map(own => own.id) })
|
|
268
|
+
// Locale sources are compiled by renderers later, so they are not part of the original's tracked input list.
|
|
269
|
+
const translations = await compileTranslations(compilation)
|
|
270
|
+
if (translations.findings.some(finding => finding.severity === 'error')) throw new RequestError(422, 'job-translations-invalid')
|
|
271
|
+
confirmed()
|
|
272
|
+
const inputs = await inputVersions([...compilation.inputs.files, ...translations.translations.flatMap(translation => translation.files)].filter(file => !compilation.sourceFiles.includes(file)))
|
|
273
|
+
confirmed()
|
|
274
|
+
// A site is built for the path its files are served at: one made below another base is not reused.
|
|
275
|
+
const inputFingerprint = fnv1a64(JSON.stringify([outputs, ...[...inputs].sort(([a], [b]) => a.localeCompare(b))]))
|
|
276
|
+
// The same target of the same revision, made or being made: not again.
|
|
277
|
+
const same = [...jobs.values()].reverse().find(job => job.document === document && job.target === target && job.revision === revision && sameSource(job.source, source) && job.inputFingerprint === inputFingerprint && ['queued', 'running', 'done'].includes(job.state))
|
|
278
|
+
return { compilation, source, revision, known, inputs, inputFingerprint, same }
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
return {
|
|
282
|
+
async targets(document) {
|
|
283
|
+
return await targetsOf(await options.compilation(document))
|
|
284
|
+
},
|
|
285
|
+
list(document) {
|
|
286
|
+
return [...jobs.values()].filter(job => document === undefined || job.document === document).reverse().map(present)
|
|
287
|
+
},
|
|
288
|
+
get(id) {
|
|
289
|
+
const job = jobs.get(id)
|
|
290
|
+
return job && present(job)
|
|
291
|
+
},
|
|
292
|
+
directory: job => (job.kind === 'render' ? join(root, job.id, job.target) : join(root, job.id)),
|
|
293
|
+
async check(document, target, expected) {
|
|
294
|
+
const admitted = await admission(document, target, expected)
|
|
295
|
+
return admitted.same && present(admitted.same)
|
|
296
|
+
},
|
|
297
|
+
async start(document, target, expected) {
|
|
298
|
+
const { compilation, source, revision, known, inputs, inputFingerprint, same } = await admission(document, target, expected)
|
|
299
|
+
if (same) {
|
|
300
|
+
options.log.info(`[job ${same.id}] document=${JSON.stringify(document || '.')} target=${target} reuse=${same.state}`)
|
|
301
|
+
return present(same)
|
|
302
|
+
}
|
|
303
|
+
const id = newId()
|
|
304
|
+
const job: CompileJob = { id, sequence: 0, document, target, kind: known.kind, state: 'queued', revision, source, inputFingerprint, createdAt: now(), url: '', files: [], warnings: [], log: [] }
|
|
305
|
+
jobs.set(id, job)
|
|
306
|
+
options.log.info(`[job ${id}] document=${JSON.stringify(document || '.')} target=${target} state=queued revision=${revision}`)
|
|
307
|
+
let resolve!: (job: CompileJob) => void
|
|
308
|
+
const entry: Running = {
|
|
309
|
+
job,
|
|
310
|
+
dir: join(root, id),
|
|
311
|
+
controller: new AbortController(),
|
|
312
|
+
ended: new Promise<CompileJob>(done => resolve = done),
|
|
313
|
+
end() {
|
|
314
|
+
running.delete(id)
|
|
315
|
+
clearTimeout(entry.timer)
|
|
316
|
+
tell(job)
|
|
317
|
+
prune(document, target)
|
|
318
|
+
resolve(present(job))
|
|
319
|
+
},
|
|
320
|
+
told: 0,
|
|
321
|
+
}
|
|
322
|
+
running.set(id, entry)
|
|
323
|
+
tell(job)
|
|
324
|
+
schedule(entry, compilation, inputs)
|
|
325
|
+
return present(job)
|
|
326
|
+
},
|
|
327
|
+
async wait(id) {
|
|
328
|
+
const entry = running.get(id)
|
|
329
|
+
if (entry) return await entry.ended
|
|
330
|
+
const job = jobs.get(id)
|
|
331
|
+
if (!job) throw new Error(`no job ${id}`)
|
|
332
|
+
return present(job)
|
|
333
|
+
},
|
|
334
|
+
cancel(id) {
|
|
335
|
+
const entry = running.get(id)
|
|
336
|
+
if (!entry) return this.get(id)
|
|
337
|
+
entry.controller.abort(new Cancelled())
|
|
338
|
+
// A job still waiting ends now; a running one when its exporter or renderer stops.
|
|
339
|
+
if (entry.job.state === 'queued') {
|
|
340
|
+
entry.job.state = 'cancelled'
|
|
341
|
+
entry.job.finishedAt = now()
|
|
342
|
+
entry.end()
|
|
343
|
+
}
|
|
344
|
+
return present(entry.job)
|
|
345
|
+
},
|
|
346
|
+
subscribe(listener) {
|
|
347
|
+
listeners.add(listener)
|
|
348
|
+
return () => listeners.delete(listener)
|
|
349
|
+
},
|
|
350
|
+
async origin() {
|
|
351
|
+
server ??= serveDirectory(root, { base: outputs, origins: options.origins })
|
|
352
|
+
return (await server).url
|
|
353
|
+
},
|
|
354
|
+
async close() {
|
|
355
|
+
closed = true
|
|
356
|
+
for (const id of [...running.keys()]) this.cancel(id)
|
|
357
|
+
await queue
|
|
358
|
+
if (server) await (await server).close()
|
|
359
|
+
},
|
|
360
|
+
}
|
|
361
|
+
}
|
package/src/node/locks.ts
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
|
-
import type { DocumentLockService } from '@lincyaw/kernel'
|
|
2
|
-
import { lockKind } from '../domain/
|
|
1
|
+
import type { DocumentLockService, LockedAnswer } from '@lincyaw/kernel'
|
|
2
|
+
import { lockKind } from '../domain/locks.ts'
|
|
3
|
+
import { RequestError } from './errors.ts'
|
|
3
4
|
|
|
4
5
|
export class DocumentLocks implements DocumentLockService {
|
|
5
6
|
readonly #held = new Map<string, string>()
|
|
6
7
|
|
|
7
8
|
claim(document: string, holder: string): (() => void) | undefined {
|
|
8
|
-
if (this
|
|
9
|
+
if (this.holder(document) !== undefined) return undefined
|
|
9
10
|
this.#held.set(document, holder)
|
|
10
11
|
let released = false
|
|
11
12
|
return () => {
|
|
@@ -16,12 +17,12 @@ export class DocumentLocks implements DocumentLockService {
|
|
|
16
17
|
}
|
|
17
18
|
|
|
18
19
|
holder(document: string): string | undefined {
|
|
19
|
-
return this.#held.get(document)
|
|
20
|
+
return this.#held.get('') ?? (document === '' ? this.#held.values().next().value : this.#held.get(document))
|
|
20
21
|
}
|
|
21
22
|
|
|
22
23
|
claimAll(documents: string[], holder: string): { release(): void } | { document: string, holder: string } {
|
|
23
24
|
const releases: (() => void)[] = []
|
|
24
|
-
for (const document of new Set(documents)) {
|
|
25
|
+
for (const document of documents.includes('') ? [''] : new Set(documents)) {
|
|
25
26
|
const release = this.claim(document, holder)
|
|
26
27
|
if (release) {
|
|
27
28
|
releases.push(release)
|
|
@@ -39,3 +40,7 @@ export function lockedMessage(document: string, holder: string): string {
|
|
|
39
40
|
const what = kind === 'agent' ? 'an agent\'s change task runs or waits for its review' : kind === 'revert' ? 'a change is being undone' : 'a document operation is under way'
|
|
40
41
|
return `${document || 'the document'} cannot change now: ${what} (${holder}); try again once it is done`
|
|
41
42
|
}
|
|
43
|
+
|
|
44
|
+
export function lockedError(document: string, holder: string): RequestError {
|
|
45
|
+
return new RequestError(409, lockedMessage(document, holder), { locked: { document, holder } } satisfies Omit<LockedAnswer, 'error'>)
|
|
46
|
+
}
|