@apollo-annotation/shared 0.1.18 → 0.1.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +13 -7
- package/src/Changes/AddAssemblyAndFeaturesFromFileChange.ts +7 -4
- package/src/Changes/AddAssemblyFromFileChange.ts +3 -2
- package/src/Changes/AddFeatureChange.ts +2 -3
- package/src/Changes/AddFeaturesFromFileChange.ts +8 -3
- package/src/Changes/AddRefSeqAliasesChange.ts +88 -0
- package/src/Changes/DeleteAssemblyChange.ts +1 -1
- package/src/Changes/DeleteFeatureChange.ts +1 -1
- package/src/Changes/FeatureAttributeChange.ts +1 -1
- package/src/Changes/FromFileBaseChange.ts +221 -0
- package/src/Changes/ImportJBrowseConfigChange.ts +116 -0
- package/src/Changes/LocationEndChange.ts +25 -18
- package/src/Changes/LocationStartChange.ts +25 -18
- package/src/Changes/TypeChange.ts +1 -1
- package/src/Changes/index.ts +6 -6
- package/src/Checks/CDSCheck.ts +251 -249
- package/src/Common/jwtPayload.ts +1 -1
- package/src/GFF3/gff3ToAnnotationFeature.test.ts +49 -0
- package/src/GFF3/gff3ToAnnotationFeature.ts +257 -0
- package/src/GFF3/gffReservedKeys.ts +61 -0
- package/src/GFF3/index.ts +2 -0
- package/src/Operations/GetFeaturesOperation.ts +2 -2
- package/src/Validations/ParentChildValidation.ts +5 -5
- package/src/index.ts +1 -0
- package/src/util.ts +15 -12
- package/tsconfig.json +4 -4
- package/src/Changes/DiscontinuousLocationEndChange.ts +0 -182
- package/src/Changes/DiscontinuousLocationStartChange.ts +0 -182
package/package.json
CHANGED
|
@@ -1,19 +1,21 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@apollo-annotation/shared",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.19",
|
|
4
4
|
"main": "./dist/index.js",
|
|
5
5
|
"scripts": {
|
|
6
6
|
"start": "tsc --build --watch",
|
|
7
7
|
"build": "yarn clean && tsc --build",
|
|
8
|
-
"clean": "rimraf dist"
|
|
8
|
+
"clean": "rimraf dist",
|
|
9
|
+
"test": "glob -c \"tsx --test --test-reporter spec \" \"**/*.test.ts\"",
|
|
10
|
+
"test:ci": "NODE_V8_COVERAGE=./coverage glob -c \"tsx --test --test-reporter spec --experimental-test-coverage \" \"**/*.test.ts\""
|
|
9
11
|
},
|
|
10
12
|
"dependencies": {
|
|
11
|
-
"@apollo-annotation/common": "^0.1.
|
|
12
|
-
"@apollo-annotation/mst": "^0.1.
|
|
13
|
-
"@apollo-annotation/schemas": "^0.1.
|
|
13
|
+
"@apollo-annotation/common": "^0.1.19",
|
|
14
|
+
"@apollo-annotation/mst": "^0.1.19",
|
|
15
|
+
"@apollo-annotation/schemas": "^0.1.19",
|
|
14
16
|
"@gmod/gff": "1.2.0",
|
|
15
17
|
"@gmod/indexedfasta": "^2.0.4",
|
|
16
|
-
"@jbrowse/core": "^2.
|
|
18
|
+
"@jbrowse/core": "^2.13.1",
|
|
17
19
|
"bson-objectid": "^2.0.4",
|
|
18
20
|
"generic-filehandle": "^3.0.0",
|
|
19
21
|
"jwt-decode": "^3.1.2",
|
|
@@ -26,11 +28,15 @@
|
|
|
26
28
|
"@nestjs/core": "^10.1.0",
|
|
27
29
|
"@types/node": "^18.14.2",
|
|
28
30
|
"@types/rimraf": "^3",
|
|
31
|
+
"glob": "^11.0.0",
|
|
29
32
|
"mobx": "^6.6.1",
|
|
30
33
|
"mobx-state-tree": "^5.1.7",
|
|
31
34
|
"mongoose": "^6.12.0",
|
|
35
|
+
"react-dom": "^18.2.0",
|
|
32
36
|
"rimraf": "^3.0.2",
|
|
33
|
-
"
|
|
37
|
+
"rxjs": "^7.4.0",
|
|
38
|
+
"tsx": "^4.6.2",
|
|
39
|
+
"typescript": "^5.5.3"
|
|
34
40
|
},
|
|
35
41
|
"peerDependencies": {
|
|
36
42
|
"@mui/material": "^5.11.14",
|
|
@@ -1,8 +1,6 @@
|
|
|
1
|
-
/* eslint-disable @typescript-eslint/restrict-template-expressions */
|
|
2
1
|
/* eslint-disable @typescript-eslint/require-await */
|
|
3
2
|
|
|
4
3
|
import {
|
|
5
|
-
AssemblySpecificChange,
|
|
6
4
|
ChangeOptions,
|
|
7
5
|
ClientDataStore,
|
|
8
6
|
LocalGFF3DataStore,
|
|
@@ -11,6 +9,8 @@ import {
|
|
|
11
9
|
} from '@apollo-annotation/common'
|
|
12
10
|
import { GFF3Feature } from '@gmod/gff'
|
|
13
11
|
|
|
12
|
+
import { FromFileBaseChange } from './FromFileBaseChange'
|
|
13
|
+
|
|
14
14
|
export interface SerializedAddAssemblyAndFeaturesFromFileChangeBase
|
|
15
15
|
extends SerializedAssemblySpecificChange {
|
|
16
16
|
typeName: 'AddAssemblyAndFeaturesFromFileChange'
|
|
@@ -34,7 +34,7 @@ export type SerializedAddAssemblyAndFeaturesFromFileChange =
|
|
|
34
34
|
| SerializedAddAssemblyAndFeaturesFromFileChangeSingle
|
|
35
35
|
| SerializedAddAssemblyAndFeaturesFromFileChangeMultiple
|
|
36
36
|
|
|
37
|
-
export class AddAssemblyAndFeaturesFromFileChange extends
|
|
37
|
+
export class AddAssemblyAndFeaturesFromFileChange extends FromFileBaseChange {
|
|
38
38
|
typeName = 'AddAssemblyAndFeaturesFromFileChange' as const
|
|
39
39
|
changes: AddAssemblyAndFeaturesFromFileChangeDetails[]
|
|
40
40
|
|
|
@@ -93,7 +93,7 @@ export class AddAssemblyAndFeaturesFromFileChange extends AssemblySpecificChange
|
|
|
93
93
|
{ _id: assembly, name: assemblyName, user, status: -1 },
|
|
94
94
|
])
|
|
95
95
|
logger.debug?.(
|
|
96
|
-
`Added new assembly "${assemblyName}", docId "${newAssemblyDoc._id}"`,
|
|
96
|
+
`Added new assembly "${assemblyName}", docId "${newAssemblyDoc._id.toHexString()}"`,
|
|
97
97
|
)
|
|
98
98
|
logger.debug?.(`File type: "${fileDoc.type}"`)
|
|
99
99
|
|
|
@@ -106,6 +106,9 @@ export class AddAssemblyAndFeaturesFromFileChange extends AssemblySpecificChange
|
|
|
106
106
|
)
|
|
107
107
|
|
|
108
108
|
// Loop all features
|
|
109
|
+
logger.debug?.(
|
|
110
|
+
`**************** LOOPATAAN KAIKKI FEATURET SEURAAVAKSI File type: "${fileDoc.type}"`,
|
|
111
|
+
)
|
|
109
112
|
const featureStream = filesService.parseGFF3(
|
|
110
113
|
filesService.getFileStream(fileDoc),
|
|
111
114
|
)
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
/* eslint-disable @typescript-eslint/restrict-template-expressions */
|
|
2
2
|
/* eslint-disable @typescript-eslint/require-await */
|
|
3
3
|
import {
|
|
4
|
-
AssemblySpecificChange,
|
|
5
4
|
ChangeOptions,
|
|
6
5
|
ClientDataStore,
|
|
7
6
|
LocalGFF3DataStore,
|
|
@@ -9,6 +8,8 @@ import {
|
|
|
9
8
|
ServerDataStore,
|
|
10
9
|
} from '@apollo-annotation/common'
|
|
11
10
|
|
|
11
|
+
import { FromFileBaseChange } from './FromFileBaseChange'
|
|
12
|
+
|
|
12
13
|
export interface SerializedAddAssemblyFromFileChangeBase
|
|
13
14
|
extends SerializedAssemblySpecificChange {
|
|
14
15
|
typeName: 'AddAssemblyFromFileChange'
|
|
@@ -32,7 +33,7 @@ export type SerializedAddAssemblyFromFileChange =
|
|
|
32
33
|
| SerializedAddAssemblyFromFileChangeSingle
|
|
33
34
|
| SerializedAddAssemblyFromFileChangeMultiple
|
|
34
35
|
|
|
35
|
-
export class AddAssemblyFromFileChange extends
|
|
36
|
+
export class AddAssemblyFromFileChange extends FromFileBaseChange {
|
|
36
37
|
typeName = 'AddAssemblyFromFileChange' as const
|
|
37
38
|
changes: AddAssemblyFromFileChangeDetails[]
|
|
38
39
|
|
|
@@ -34,7 +34,7 @@ interface SerializedAddFeatureChangeMultiple
|
|
|
34
34
|
changes: AddFeatureChangeDetails[]
|
|
35
35
|
}
|
|
36
36
|
|
|
37
|
-
type SerializedAddFeatureChange =
|
|
37
|
+
export type SerializedAddFeatureChange =
|
|
38
38
|
| SerializedAddFeatureChangeSingle
|
|
39
39
|
| SerializedAddFeatureChangeMultiple
|
|
40
40
|
|
|
@@ -113,7 +113,6 @@ export class AddFeatureChange extends FeatureChange {
|
|
|
113
113
|
)
|
|
114
114
|
featureCnt++
|
|
115
115
|
} else {
|
|
116
|
-
addedFeature.gffId = _id // User added manually new feature so then gffId = _id
|
|
117
116
|
// Adding new child feature
|
|
118
117
|
if (parentFeatureId) {
|
|
119
118
|
const topLevelFeature = await featureModel
|
|
@@ -158,7 +157,7 @@ export class AddFeatureChange extends FeatureChange {
|
|
|
158
157
|
// Child features should be sorted for click and drag of gene glyphs to work properly
|
|
159
158
|
parentFeature.children = new Map(
|
|
160
159
|
[...parentFeature.children.entries()].sort(
|
|
161
|
-
(a, b) => a[1].
|
|
160
|
+
(a, b) => a[1].min - b[1].min,
|
|
162
161
|
),
|
|
163
162
|
)
|
|
164
163
|
const childIds = this.getChildFeatureIds(addedFeature)
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
/* eslint-disable @typescript-eslint/require-await */
|
|
2
2
|
import {
|
|
3
|
-
AssemblySpecificChange,
|
|
4
3
|
ChangeOptions,
|
|
5
4
|
ClientDataStore,
|
|
6
5
|
LocalGFF3DataStore,
|
|
@@ -9,6 +8,8 @@ import {
|
|
|
9
8
|
} from '@apollo-annotation/common'
|
|
10
9
|
import { GFF3Feature } from '@gmod/gff'
|
|
11
10
|
|
|
11
|
+
import { FromFileBaseChange } from './FromFileBaseChange'
|
|
12
|
+
|
|
12
13
|
export interface SerializedAddFeaturesFromFileChangeBase
|
|
13
14
|
extends SerializedAssemblySpecificChange {
|
|
14
15
|
typeName: 'AddFeaturesFromFileChange'
|
|
@@ -32,7 +33,7 @@ export type SerializedAddFeaturesFromFileChange =
|
|
|
32
33
|
| SerializedAddFeaturesFromFileChangeSingle
|
|
33
34
|
| SerializedAddFeaturesFromFileChangeMultiple
|
|
34
35
|
|
|
35
|
-
export class AddFeaturesFromFileChange extends
|
|
36
|
+
export class AddFeaturesFromFileChange extends FromFileBaseChange {
|
|
36
37
|
typeName = 'AddFeaturesFromFileChange' as const
|
|
37
38
|
changes: AddFeaturesFromFileChangeDetails[]
|
|
38
39
|
deleteExistingFeatures = false
|
|
@@ -91,13 +92,17 @@ export class AddFeaturesFromFileChange extends AssemblySpecificChange {
|
|
|
91
92
|
const featureStream = filesService.parseGFF3(
|
|
92
93
|
filesService.getFileStream(fileDoc),
|
|
93
94
|
)
|
|
95
|
+
let featureCount = 0
|
|
94
96
|
for await (const f of featureStream) {
|
|
95
97
|
const gff3Feature = f as GFF3Feature
|
|
96
|
-
logger.verbose?.(`ENTRY=${JSON.stringify(gff3Feature)}`)
|
|
97
98
|
|
|
98
99
|
// Add new feature into database
|
|
99
100
|
// We cannot use Mongo 'session' / transaction here because Mongo has 16 MB limit for transaction
|
|
100
101
|
await this.addFeatureIntoDb(gff3Feature, backend)
|
|
102
|
+
featureCount++
|
|
103
|
+
if (featureCount % 1000 === 0) {
|
|
104
|
+
logger.debug?.(`Processed ${featureCount} features`)
|
|
105
|
+
}
|
|
101
106
|
}
|
|
102
107
|
}
|
|
103
108
|
logger.debug?.('New features added into database!')
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import {
|
|
2
|
+
AssemblySpecificChange,
|
|
3
|
+
Change,
|
|
4
|
+
ChangeOptions,
|
|
5
|
+
ClientDataStore,
|
|
6
|
+
LocalGFF3DataStore,
|
|
7
|
+
SerializedAssemblySpecificChange,
|
|
8
|
+
ServerDataStore,
|
|
9
|
+
} from '@apollo-annotation/common'
|
|
10
|
+
import { getSession } from '@jbrowse/core/util'
|
|
11
|
+
|
|
12
|
+
export interface SerializedRefSeqAliases {
|
|
13
|
+
refName: string
|
|
14
|
+
aliases: string[]
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export interface SerializedRefSeqAliasesChange
|
|
18
|
+
extends SerializedAssemblySpecificChange {
|
|
19
|
+
typeName: 'AddRefSeqAliasesChange'
|
|
20
|
+
refSeqAliases: SerializedRefSeqAliases[]
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export class AddRefSeqAliasesChange extends AssemblySpecificChange {
|
|
24
|
+
typeName = 'AddRefSeqAliasesChange' as const
|
|
25
|
+
refSeqAliases: SerializedRefSeqAliases[]
|
|
26
|
+
|
|
27
|
+
constructor(json: SerializedRefSeqAliasesChange, options?: ChangeOptions) {
|
|
28
|
+
super(json, options)
|
|
29
|
+
this.refSeqAliases = json.refSeqAliases
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
executeOnClient(clientDataStore: ClientDataStore) {
|
|
33
|
+
const { assemblyManager } = getSession(clientDataStore)
|
|
34
|
+
const assembly = assemblyManager.get(this.assembly)
|
|
35
|
+
if (!assembly) {
|
|
36
|
+
throw new Error(`assembly ${this.assembly} not found`)
|
|
37
|
+
}
|
|
38
|
+
const sessionAliases = assembly.refNameAliases
|
|
39
|
+
const sessionLCAliases = assembly.lowerCaseRefNameAliases
|
|
40
|
+
|
|
41
|
+
if (!sessionAliases || !sessionLCAliases) {
|
|
42
|
+
throw new Error('Session refNameAliases not found in assembly')
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
for (const refSeqAlias of this.refSeqAliases) {
|
|
46
|
+
const { aliases, refName } = refSeqAlias
|
|
47
|
+
for (const alias of aliases) {
|
|
48
|
+
sessionAliases[alias] = refName
|
|
49
|
+
sessionLCAliases[alias.toLowerCase()] = refName
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
assembly.setRefNameAliases(sessionAliases, sessionLCAliases)
|
|
53
|
+
return Promise.resolve()
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
getInverse(): Change {
|
|
57
|
+
throw new Error('Method not implemented.')
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
toJSON(): SerializedRefSeqAliasesChange {
|
|
61
|
+
const { assembly, refSeqAliases, typeName } = this
|
|
62
|
+
return { assembly, typeName, refSeqAliases }
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
async executeOnServer(backend: ServerDataStore) {
|
|
66
|
+
const { refSeqModel, session } = backend
|
|
67
|
+
const { assembly, logger, refSeqAliases } = this
|
|
68
|
+
|
|
69
|
+
for (const refSeqAlias of refSeqAliases) {
|
|
70
|
+
logger.debug?.(
|
|
71
|
+
`Updating Refname alias for assembly: ${assembly}, refSeqAlias: ${JSON.stringify(refSeqAlias)}`,
|
|
72
|
+
)
|
|
73
|
+
const { aliases, refName } = refSeqAlias
|
|
74
|
+
await refSeqModel
|
|
75
|
+
.updateOne({ assembly, name: refName }, { $set: { aliases } })
|
|
76
|
+
.session(session)
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
executeOnLocalGFF3(_backend: LocalGFF3DataStore): Promise<unknown> {
|
|
81
|
+
throw new Error('Method not implemented.')
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
// eslint-disable-next-line @typescript-eslint/class-literal-property-style
|
|
85
|
+
get notification(): string {
|
|
86
|
+
return 'RefSeq aliases have been added.'
|
|
87
|
+
}
|
|
88
|
+
}
|
|
@@ -11,7 +11,7 @@ import {
|
|
|
11
11
|
} from '@apollo-annotation/common'
|
|
12
12
|
import { getSession } from '@jbrowse/core/util'
|
|
13
13
|
|
|
14
|
-
interface SerializedDeleteAssemblyChange
|
|
14
|
+
export interface SerializedDeleteAssemblyChange
|
|
15
15
|
extends SerializedAssemblySpecificChange {
|
|
16
16
|
typeName: 'DeleteAssemblyChange'
|
|
17
17
|
}
|
|
@@ -32,7 +32,7 @@ interface SerializedDeleteFeatureChangeMultiple
|
|
|
32
32
|
changes: DeleteFeatureChangeDetails[]
|
|
33
33
|
}
|
|
34
34
|
|
|
35
|
-
type SerializedDeleteFeatureChange =
|
|
35
|
+
export type SerializedDeleteFeatureChange =
|
|
36
36
|
| SerializedDeleteFeatureChangeSingle
|
|
37
37
|
| SerializedDeleteFeatureChangeMultiple
|
|
38
38
|
|
|
@@ -29,7 +29,7 @@ interface SerializedFeatureAttributeChangeMultiple
|
|
|
29
29
|
changes: FeatureAttributeChangeDetails[]
|
|
30
30
|
}
|
|
31
31
|
|
|
32
|
-
type SerializedFeatureAttributeChange =
|
|
32
|
+
export type SerializedFeatureAttributeChange =
|
|
33
33
|
| SerializedFeatureAttributeChangeSingle
|
|
34
34
|
| SerializedFeatureAttributeChangeMultiple
|
|
35
35
|
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
/* eslint-disable @typescript-eslint/restrict-plus-operands */
|
|
2
|
+
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
|
|
3
|
+
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
|
|
4
|
+
/* eslint-disable @typescript-eslint/no-unsafe-call */
|
|
5
|
+
/* eslint-disable @typescript-eslint/no-unsafe-argument */
|
|
6
|
+
import {
|
|
7
|
+
AssemblySpecificChange,
|
|
8
|
+
ServerDataStore,
|
|
9
|
+
} from '@apollo-annotation/common'
|
|
10
|
+
import { FileDocument, RefSeqDocument } from '@apollo-annotation/schemas'
|
|
11
|
+
import { GFF3Feature } from '@gmod/gff'
|
|
12
|
+
|
|
13
|
+
import { gff3ToAnnotationFeature } from '../GFF3'
|
|
14
|
+
|
|
15
|
+
export abstract class FromFileBaseChange extends AssemblySpecificChange {
|
|
16
|
+
async addRefSeqIntoDb(
|
|
17
|
+
fileDoc: FileDocument,
|
|
18
|
+
assembly: string,
|
|
19
|
+
backend: ServerDataStore,
|
|
20
|
+
) {
|
|
21
|
+
const { logger } = this
|
|
22
|
+
const { filesService, refSeqChunkModel, refSeqModel, user } = backend
|
|
23
|
+
const { CHUNK_SIZE } = process.env
|
|
24
|
+
const customChunkSize = CHUNK_SIZE && Number(CHUNK_SIZE)
|
|
25
|
+
let chunkIndex = 0
|
|
26
|
+
let refSeqLen = 0
|
|
27
|
+
let refSeqDoc: RefSeqDocument | undefined
|
|
28
|
+
let fastaInfoStarted = fileDoc.type !== 'text/x-gff3'
|
|
29
|
+
|
|
30
|
+
// Read data from compressed file and parse the content
|
|
31
|
+
const sequenceStream = filesService.getFileStream(fileDoc)
|
|
32
|
+
let sequenceBuffer = ''
|
|
33
|
+
let incompleteLine = ''
|
|
34
|
+
let lastLineIsIncomplete = true
|
|
35
|
+
let parsingStarted = false
|
|
36
|
+
logger.debug?.('starting sequence stream')
|
|
37
|
+
let lineCount = 0
|
|
38
|
+
for await (const data of sequenceStream) {
|
|
39
|
+
const chunk = data.toString()
|
|
40
|
+
lastLineIsIncomplete = !chunk.endsWith('\n')
|
|
41
|
+
// chunk is small enough that you can split the whole thing into lines without having to make it into smaller chunks first.
|
|
42
|
+
const lines = chunk.split(/\r?\n/)
|
|
43
|
+
if (incompleteLine) {
|
|
44
|
+
lines[0] = `${incompleteLine}${lines[0]}`
|
|
45
|
+
incompleteLine = ''
|
|
46
|
+
}
|
|
47
|
+
if (lastLineIsIncomplete) {
|
|
48
|
+
incompleteLine = lines.pop() || ''
|
|
49
|
+
}
|
|
50
|
+
for await (const line of lines) {
|
|
51
|
+
lineCount++
|
|
52
|
+
if (lineCount % 1_000_000 === 0) {
|
|
53
|
+
logger.debug?.(`Processed ${lineCount} lines`)
|
|
54
|
+
}
|
|
55
|
+
// In case of GFF3 file we start to read sequence after '##FASTA' is found
|
|
56
|
+
if (!fastaInfoStarted) {
|
|
57
|
+
if (line.trim() === '##FASTA') {
|
|
58
|
+
fastaInfoStarted = true
|
|
59
|
+
}
|
|
60
|
+
continue
|
|
61
|
+
}
|
|
62
|
+
const refSeqInfoLine = /^>\s*(\S+)\s*(.*)/.exec(line)
|
|
63
|
+
// Add new ref sequence info if we are reference seq info line
|
|
64
|
+
if (refSeqInfoLine) {
|
|
65
|
+
parsingStarted = true
|
|
66
|
+
logger.debug?.(
|
|
67
|
+
`Reference sequence information line "${refSeqInfoLine[0]}"`,
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
// If there is sequence from previous reference sequence then we need to add it to previous ref seq
|
|
71
|
+
if (sequenceBuffer !== '') {
|
|
72
|
+
if (!refSeqDoc) {
|
|
73
|
+
throw new Error('No refSeq document found')
|
|
74
|
+
}
|
|
75
|
+
refSeqLen += sequenceBuffer.length
|
|
76
|
+
// We cannot use Mongo 'session' / transaction here because Mongo has 16 MB limit for transaction
|
|
77
|
+
await refSeqChunkModel.create([
|
|
78
|
+
{
|
|
79
|
+
refSeq: refSeqDoc._id,
|
|
80
|
+
n: chunkIndex,
|
|
81
|
+
sequence: sequenceBuffer,
|
|
82
|
+
user,
|
|
83
|
+
status: -1,
|
|
84
|
+
},
|
|
85
|
+
])
|
|
86
|
+
sequenceBuffer = ''
|
|
87
|
+
}
|
|
88
|
+
await refSeqDoc?.updateOne({ length: refSeqLen })
|
|
89
|
+
// await refSeqDoc?.updateOne({ length: refSeqLen }, { session })
|
|
90
|
+
refSeqLen = 0
|
|
91
|
+
chunkIndex = 0
|
|
92
|
+
|
|
93
|
+
const name = refSeqInfoLine[1].trim()
|
|
94
|
+
const description = refSeqInfoLine[2] ? refSeqInfoLine[2].trim() : ''
|
|
95
|
+
|
|
96
|
+
// We cannot use Mongo 'session' / transaction here because Mongo has 16 MB limit for transaction
|
|
97
|
+
const [newRefSeqDoc] = await refSeqModel.create([
|
|
98
|
+
{
|
|
99
|
+
name,
|
|
100
|
+
description,
|
|
101
|
+
assembly,
|
|
102
|
+
length: 0,
|
|
103
|
+
...(customChunkSize ? { chunkSize: customChunkSize } : null),
|
|
104
|
+
user,
|
|
105
|
+
status: -1,
|
|
106
|
+
},
|
|
107
|
+
])
|
|
108
|
+
logger.debug?.(
|
|
109
|
+
`Added new refSeq "${name}", desc "${description}", docId "${newRefSeqDoc._id}"`,
|
|
110
|
+
)
|
|
111
|
+
refSeqDoc = newRefSeqDoc
|
|
112
|
+
} else if (/\S/.test(line)) {
|
|
113
|
+
if (!refSeqDoc) {
|
|
114
|
+
throw new Error('No refSeq document found')
|
|
115
|
+
}
|
|
116
|
+
const { _id, chunkSize } = refSeqDoc
|
|
117
|
+
sequenceBuffer += line.replaceAll(/\s/g, '')
|
|
118
|
+
// If sequence block > chunk size then save chunk into Mongo
|
|
119
|
+
while (sequenceBuffer.length >= chunkSize) {
|
|
120
|
+
const sequence = sequenceBuffer.slice(0, chunkSize)
|
|
121
|
+
refSeqLen += sequence.length
|
|
122
|
+
// We cannot use Mongo 'session' / transaction here because Mongo has 16 MB limit for transaction
|
|
123
|
+
await refSeqChunkModel.create([
|
|
124
|
+
{ refSeq: _id, n: chunkIndex, sequence, user, status: -1 },
|
|
125
|
+
])
|
|
126
|
+
chunkIndex++
|
|
127
|
+
// Set remaining sequence
|
|
128
|
+
sequenceBuffer = sequenceBuffer.slice(chunkSize)
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
if (!parsingStarted) {
|
|
134
|
+
throw new Error('No reference sequences found in file')
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
if (sequenceBuffer || lastLineIsIncomplete) {
|
|
138
|
+
if (!refSeqDoc) {
|
|
139
|
+
throw new Error('No refSeq document found')
|
|
140
|
+
}
|
|
141
|
+
// If the file did not end with line break so the last line is incomplete
|
|
142
|
+
if (lastLineIsIncomplete) {
|
|
143
|
+
sequenceBuffer += incompleteLine
|
|
144
|
+
}
|
|
145
|
+
refSeqLen += sequenceBuffer.length
|
|
146
|
+
logger.verbose?.(
|
|
147
|
+
`*** Add the very last chunk to ref seq ("${refSeqDoc._id}", index ${chunkIndex} and total length for ref seq is ${refSeqLen}): "${sequenceBuffer}"`,
|
|
148
|
+
)
|
|
149
|
+
logger.debug?.(
|
|
150
|
+
`Creating refSeq chunk number ${chunkIndex} of "${refSeqDoc._id}"`,
|
|
151
|
+
)
|
|
152
|
+
// We cannot use Mongo 'session' / transaction here because Mongo has 16 MB limit for transaction
|
|
153
|
+
await refSeqChunkModel.create([
|
|
154
|
+
{
|
|
155
|
+
refSeq: refSeqDoc._id,
|
|
156
|
+
n: chunkIndex,
|
|
157
|
+
sequence: sequenceBuffer,
|
|
158
|
+
user,
|
|
159
|
+
status: -1,
|
|
160
|
+
},
|
|
161
|
+
])
|
|
162
|
+
await refSeqDoc.updateOne({ length: refSeqLen })
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
private refSeqCache = new Map<string, RefSeqDocument>()
|
|
167
|
+
|
|
168
|
+
async removeExistingFeatures(backend: ServerDataStore) {
|
|
169
|
+
const { featureModel, refSeqModel } = backend
|
|
170
|
+
const { assembly, logger } = this
|
|
171
|
+
logger.debug?.(`Removing existing features for assembly = ${assembly}`)
|
|
172
|
+
|
|
173
|
+
const refSeqs: RefSeqDocument[] = await refSeqModel
|
|
174
|
+
.find({ assembly })
|
|
175
|
+
.exec()
|
|
176
|
+
|
|
177
|
+
for (const refSeq of refSeqs) {
|
|
178
|
+
await featureModel.deleteMany({ refSeq: refSeq._id })
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
async addFeatureIntoDb(gff3Feature: GFF3Feature, backend: ServerDataStore) {
|
|
183
|
+
const { featureModel, refSeqModel, user } = backend
|
|
184
|
+
const { assembly, refSeqCache } = this
|
|
185
|
+
|
|
186
|
+
const [{ seq_id: refName }] = gff3Feature
|
|
187
|
+
if (!refName) {
|
|
188
|
+
throw new Error(
|
|
189
|
+
`Valid seq_id not found in feature ${JSON.stringify(gff3Feature)}`,
|
|
190
|
+
)
|
|
191
|
+
}
|
|
192
|
+
let refSeqDoc = refSeqCache.get(refName)
|
|
193
|
+
if (!refSeqDoc) {
|
|
194
|
+
refSeqDoc =
|
|
195
|
+
(await refSeqModel.findOne({ assembly, name: refName }).exec()) ??
|
|
196
|
+
undefined
|
|
197
|
+
if (refSeqDoc) {
|
|
198
|
+
refSeqCache.set(refName, refSeqDoc)
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
if (!refSeqDoc) {
|
|
202
|
+
throw new Error(
|
|
203
|
+
`RefSeq was not found by assembly "${assembly}" and seq_id "${refName}" not found`,
|
|
204
|
+
)
|
|
205
|
+
}
|
|
206
|
+
// Let's add featureId to parent feature
|
|
207
|
+
const featureIds: string[] = []
|
|
208
|
+
|
|
209
|
+
const newFeature = gff3ToAnnotationFeature(
|
|
210
|
+
gff3Feature,
|
|
211
|
+
refSeqDoc._id,
|
|
212
|
+
featureIds,
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
// Add into Mongo
|
|
216
|
+
// We cannot use Mongo 'session' / transaction here because Mongo has 16 MB limit for transaction
|
|
217
|
+
await featureModel.create([
|
|
218
|
+
{ allIds: featureIds, ...newFeature, user, status: -1 },
|
|
219
|
+
])
|
|
220
|
+
}
|
|
221
|
+
}
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
/* eslint-disable @typescript-eslint/require-await */
|
|
2
|
+
import {
|
|
3
|
+
Change,
|
|
4
|
+
ChangeOptions,
|
|
5
|
+
ClientDataStore,
|
|
6
|
+
LocalGFF3DataStore,
|
|
7
|
+
ServerDataStore,
|
|
8
|
+
} from '@apollo-annotation/common'
|
|
9
|
+
|
|
10
|
+
interface JBrowseAssembly {
|
|
11
|
+
sequence: { adapter: { type: string } }
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
interface JBrowseTrack {
|
|
15
|
+
type: string
|
|
16
|
+
trackId: string
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
interface JBrowsePlugin {
|
|
20
|
+
name: string
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
interface JBrowseInternetAccount {
|
|
24
|
+
type: string
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export interface JBrowseConfig {
|
|
28
|
+
assemblies?: JBrowseAssembly[]
|
|
29
|
+
tracks?: JBrowseTrack[]
|
|
30
|
+
plugins?: JBrowsePlugin[]
|
|
31
|
+
internetAccounts?: JBrowseInternetAccount[]
|
|
32
|
+
[key: string]: unknown
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export interface SerializedImportJBrowseConfigChange {
|
|
36
|
+
typeName: 'ImportJBrowseConfigChange'
|
|
37
|
+
oldJBrowseConfig?: JBrowseConfig
|
|
38
|
+
newJBrowseConfig?: JBrowseConfig
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export function filterJBrowseConfig(config: JBrowseConfig): JBrowseConfig {
|
|
42
|
+
const { assemblies, internetAccounts, plugins, tracks, ...rest } = config
|
|
43
|
+
const filteredAssemblies = assemblies?.filter(
|
|
44
|
+
(a) => a.sequence.adapter.type !== 'ApolloSequenceAdapter',
|
|
45
|
+
)
|
|
46
|
+
const filteredTracks = tracks?.filter((t) => t.type !== 'ApolloTrack')
|
|
47
|
+
const filteredPlugins = plugins?.filter((p) => p.name !== 'Apollo')
|
|
48
|
+
const filteredInternetAccounts = internetAccounts?.filter(
|
|
49
|
+
(i) => i.type !== 'ApolloInternetAccount',
|
|
50
|
+
)
|
|
51
|
+
const filteredConfig = rest as JBrowseConfig
|
|
52
|
+
if (filteredAssemblies) {
|
|
53
|
+
filteredConfig.assemblies = filteredAssemblies
|
|
54
|
+
}
|
|
55
|
+
if (filteredTracks) {
|
|
56
|
+
filteredConfig.tracks = filteredTracks
|
|
57
|
+
}
|
|
58
|
+
if (filteredPlugins) {
|
|
59
|
+
filteredConfig.plugins = filteredPlugins
|
|
60
|
+
}
|
|
61
|
+
if (filteredInternetAccounts) {
|
|
62
|
+
filteredConfig.internetAccounts = filteredInternetAccounts
|
|
63
|
+
}
|
|
64
|
+
return filteredConfig
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export class ImportJBrowseConfigChange extends Change {
|
|
68
|
+
typeName = 'ImportJBrowseConfigChange' as const
|
|
69
|
+
oldJBrowseConfig?: JBrowseConfig
|
|
70
|
+
newJBrowseConfig?: JBrowseConfig
|
|
71
|
+
|
|
72
|
+
constructor(
|
|
73
|
+
json: SerializedImportJBrowseConfigChange,
|
|
74
|
+
options?: ChangeOptions,
|
|
75
|
+
) {
|
|
76
|
+
super(json, options)
|
|
77
|
+
this.oldJBrowseConfig = json.oldJBrowseConfig
|
|
78
|
+
this.newJBrowseConfig = json.newJBrowseConfig
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
toJSON(): SerializedImportJBrowseConfigChange {
|
|
82
|
+
const { newJBrowseConfig, oldJBrowseConfig, typeName } = this
|
|
83
|
+
return { typeName, oldJBrowseConfig, newJBrowseConfig }
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
async executeOnServer(backend: ServerDataStore) {
|
|
87
|
+
const { jbrowseConfigModel } = backend
|
|
88
|
+
const { logger, newJBrowseConfig } = this
|
|
89
|
+
await jbrowseConfigModel.deleteMany()
|
|
90
|
+
if (!newJBrowseConfig) {
|
|
91
|
+
return
|
|
92
|
+
}
|
|
93
|
+
const filteredConfig = filterJBrowseConfig(newJBrowseConfig)
|
|
94
|
+
await jbrowseConfigModel.create(filteredConfig)
|
|
95
|
+
logger.debug?.('Stored new JBrowse Config')
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
async executeOnLocalGFF3(_backend: LocalGFF3DataStore) {
|
|
99
|
+
throw new Error('executeOnLocalGFF3 not implemented')
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// eslint-disable-next-line @typescript-eslint/no-empty-function
|
|
103
|
+
async executeOnClient(_dataStore: ClientDataStore) {}
|
|
104
|
+
|
|
105
|
+
getInverse() {
|
|
106
|
+
const { logger, newJBrowseConfig, oldJBrowseConfig } = this
|
|
107
|
+
return new ImportJBrowseConfigChange(
|
|
108
|
+
{
|
|
109
|
+
typeName: 'ImportJBrowseConfigChange',
|
|
110
|
+
oldJBrowseConfig: newJBrowseConfig,
|
|
111
|
+
newJBrowseConfig: oldJBrowseConfig,
|
|
112
|
+
},
|
|
113
|
+
{ logger },
|
|
114
|
+
)
|
|
115
|
+
}
|
|
116
|
+
}
|