cloudcms-server 0.9.262 → 0.9.263

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (9) hide show
  1. package/locks/locks.js +49 -4
  2. package/locks/providers/memory.js +9 -6
  3. package/locks/providers/redis.js +1 -1
  4. package/middleware/awareness/providers/abstract-async.js +33 -26
  5. package/middleware/cloudcms/cloudcms.js +17 -15
  6. package/package.json +2 -2
  7. package/temp/clusterlock/index.js +3 -3
  8. package/temp/clusterlock/package.json +1 -1
  9. package/util/cloudcms.js +65 -86
package/locks/locks.js CHANGED
@@ -25,6 +25,8 @@ var async = require("async");
25
25
  */
26
26
  exports = module.exports = function()
27
27
  {
28
+ var lockTimeoutMs = 120000; // two minutes
29
+
28
30
  var provider = null;
29
31
 
30
32
  var r = {};
@@ -72,12 +74,55 @@ exports = module.exports = function()
72
74
  */
73
75
  var lock = r.lock = function(key, fn)
74
76
  {
75
- provider.lock(key, function(err, releaseFn) {
77
+ var __log = function(key, text) {
78
+ // var skip = false;
79
+ // if (key === "channels") { skip = true; }
80
+ // if (!skip) {
81
+ // console.log("[LOCK: " + key + "] " + text);
82
+ // }
83
+ };
84
+
85
+ __log(key, "request");
86
+ provider.lock(key, function(err, _releaseFn) {
87
+
88
+ // wrap the releaseFn with a wrapper that can only fire once
89
+ var releaseFn = function(_releaseFn)
90
+ {
91
+ var triggered = false;
92
+ return function() {
93
+ if (!triggered) {
94
+ triggered = true;
95
+ _releaseFn();
96
+ return true;
97
+ }
98
+
99
+ return false;
100
+ }
101
+ }(_releaseFn);
102
+
103
+ // after 120 seconds, we force release lock (if it hasn't already been released)
104
+ (function(key, releaseFn) {
105
+ setTimeout(function() {
106
+ var released = releaseFn();
107
+ if (released) {
108
+ __log(key, "timed out, released");
109
+ }
110
+ }, lockTimeoutMs);
111
+ })(key, releaseFn);
112
+
113
+ __log(key, "taken");
114
+
76
115
  fn(err, function(afterReleaseCallback) {
116
+ __log(key, "pre-release");
77
117
 
78
- releaseFn();
79
-
80
- if (afterReleaseCallback)
118
+ var released = releaseFn();
119
+ if (released) {
120
+ __log(key, "released");
121
+ } else {
122
+ __log(key, "not released, was previously released on timeout");
123
+ }
124
+
125
+ if (released && afterReleaseCallback)
81
126
  {
82
127
  afterReleaseCallback();
83
128
  }
package/locks/providers/memory.js CHANGED
@@ -1,6 +1,4 @@
1
- var path = require("path");
2
-
3
- var ReadWriteLock = require("rwlock");
1
+ var AsyncLock = require('async-lock');
4
2
 
5
3
  /**
6
4
  * Simple in-memory lock service.
@@ -12,8 +10,8 @@ var ReadWriteLock = require("rwlock");
12
10
  exports = module.exports = function(lockConfig)
13
11
  {
14
12
  var r = {};
15
-
16
- var locker = new ReadWriteLock();
13
+
14
+ var lock = new AsyncLock();
17
15
 
18
16
  r.init = function(callback)
19
17
  {
@@ -22,8 +20,13 @@ exports = module.exports = function(lockConfig)
22
20
 
23
21
  r.lock = function(key, fn)
24
22
  {
25
- locker.writeLock(key, function(releaseCallbackFn) {
23
+ lock.acquire(key, function(releaseCallbackFn) {
26
24
  fn(null, releaseCallbackFn);
25
+ }, function(err, ret) {
26
+ // lock was released
27
+ if (err) {
28
+ console.error("Memory Lock heard error: ", err, " return value: ", ret);
29
+ }
27
30
  });
28
31
  };
29
32
 
package/locks/providers/redis.js CHANGED
@@ -13,7 +13,7 @@ exports = module.exports = function(locksConfig)
13
13
  var redlock = null;
14
14
  var client = null;
15
15
 
16
- var logger = redisHelper.redisLogger("REDIS_LOCKS", "CLOUDCMS_LOCKS_", "error")
16
+ var logger = redisHelper.redisLogger("REDIS_LOCKS", "CLOUDCMS_LOCKS_", "debug")
17
17
 
18
18
  var r = {};
19
19
 
package/middleware/awareness/providers/abstract-async.js CHANGED
@@ -52,9 +52,14 @@ class AbstractAsyncProvider extends AbstractProvider
52
52
  {
53
53
  var self = this;
54
54
 
55
+ //console.log("a1");
55
56
  self._lock("channels", function(err, releaseLockFn) {
57
+ // console.log("a2: ", err);
58
+ // console.log("a3: ", releaseLockFn);
56
59
  self.doRegister(channelId, user, function(err) {
60
+ // console.log("a4 ", err);
57
61
  callback(err);
62
+ // console.log("a5: ", releaseLockFn);
58
63
  return releaseLockFn();
59
64
  });
60
65
  });
@@ -206,38 +211,40 @@ class AbstractAsyncProvider extends AbstractProvider
206
211
  return done();
207
212
  }
208
213
 
209
- if (channel.users && Object.keys(channel.users).length > 0)
214
+ if (!channel.users || Object.keys(channel.users).length === 0)
210
215
  {
211
- // populate all of the user IDs that need to be removed
212
- var userIdsToRemove = [];
213
- for (var userId in channel.users)
216
+ return done();
217
+ }
218
+
219
+ // populate all of the user IDs that need to be removed
220
+ var userIdsToRemove = [];
221
+ for (var userId in channel.users)
222
+ {
223
+ var entry = channel.users[userId];
224
+ if (entry.time < beforeMs)
214
225
  {
215
- var entry = channel.users[userId];
216
- if (entry.time < beforeMs)
217
- {
218
- updatedMembershipChannelIds.push(channelId);
219
- userIdsToRemove.push(userId);
220
-
221
- var expiredUserIds = expiredUserIdsByChannelId[channelId]
222
- if (!expiredUserIds) {
223
- expiredUserIds = expiredUserIdsByChannelId[channelId] = [];
224
- }
225
-
226
- expiredUserIds.push(userId);
226
+ updatedMembershipChannelIds.push(channelId);
227
+ userIdsToRemove.push(userId);
228
+
229
+ var expiredUserIds = expiredUserIdsByChannelId[channelId]
230
+ if (!expiredUserIds) {
231
+ expiredUserIds = expiredUserIdsByChannelId[channelId] = [];
227
232
  }
233
+
234
+ expiredUserIds.push(userId);
228
235
  }
229
-
230
- // remove the user IDs
231
- for (var i = 0; i < userIdsToRemove.length; i++)
232
- {
233
- delete channel.users[userIdsToRemove[i]];
234
- }
235
-
236
- self.writeChannel(channelId, channel, function() {
237
- done();
238
- });
239
236
  }
240
237
 
238
+ // remove the user IDs
239
+ for (var i = 0; i < userIdsToRemove.length; i++)
240
+ {
241
+ delete channel.users[userIdsToRemove[i]];
242
+ }
243
+
244
+ self.writeChannel(channelId, channel, function() {
245
+ done();
246
+ });
247
+
241
248
  });
242
249
  };
243
250
  }(channelId, updatedMembershipChannelIds, expiredUserIdsByChannelId, beforeMs);
package/middleware/cloudcms/cloudcms.js CHANGED
@@ -441,7 +441,13 @@ exports = module.exports = function()
441
441
  if (err) {
442
442
  return callback(err);
443
443
  }
444
-
444
+
445
+ var branch = CACHED_BRANCHES[cacheKey];
446
+ if (branch) {
447
+ callback(null, Chain(branch));
448
+ return releaseLockFn();
449
+ }
450
+
445
451
  var loadFn = function(finished) {
446
452
 
447
453
  Chain(repository).trap(function(e) {
@@ -468,23 +474,21 @@ exports = module.exports = function()
468
474
 
469
475
  if (err) {
470
476
 
477
+ callback(err);
478
+
471
479
  // release the lock
472
- releaseLockFn();
473
-
474
- // do the callback
475
- return callback(err);
480
+ return releaseLockFn();
476
481
  }
477
482
 
478
483
  // success!
479
484
 
480
485
  // store in cache
481
486
  CACHED_BRANCHES[cacheKey] = branch;
482
-
487
+
488
+ callback(null, branch);
489
+
483
490
  // release the lock
484
- releaseLockFn();
485
-
486
- // do the callback
487
- return callback(null, branch);
491
+ return releaseLockFn();
488
492
  });
489
493
  }
490
494
 
@@ -1443,11 +1447,10 @@ exports = module.exports = function()
1443
1447
 
1444
1448
  // the range requested (for streaming)
1445
1449
  //var range = req.headers["range"];
1446
-
1450
+
1447
1451
  cloudcmsUtil.preview(contentStore, gitana, repositoryId, branchId, nodeId, nodePath, attachmentId, locale, previewId, size, mimetype, forceReload, function(err, filePath, cacheInfo, releaseLock) {
1448
1452
 
1449
- if (err)
1450
- {
1453
+ if (err) {
1451
1454
  req.log("Error on preview node: " + err.message + ", err: " + JSON.stringify(err));
1452
1455
  }
1453
1456
 
@@ -1479,8 +1482,7 @@ exports = module.exports = function()
1479
1482
  // UZI: file deleted by invalidate before this gets called
1480
1483
  contentStore.sendFile(res, filePath, cacheInfo, function(err) {
1481
1484
 
1482
- if (err)
1483
- {
1485
+ if (err) {
1484
1486
  util.handleSendFileError(req, res, filePath, cacheInfo, req.log, err);
1485
1487
  }
1486
1488
 
package/package.json CHANGED
@@ -6,7 +6,7 @@
6
6
  },
7
7
  "name": "cloudcms-server",
8
8
  "description": "Cloud CMS Application Server Module",
9
- "version": "0.9.262",
9
+ "version": "0.9.263",
10
10
  "repository": {
11
11
  "type": "git",
12
12
  "url": "git://github.com/gitana/cloudcms-server.git"
@@ -20,6 +20,7 @@
20
20
  "alpaca": "^1.5.27",
21
21
  "archiver": "^1.3.0",
22
22
  "async": "^3.2.3",
23
+ "async-lock": "^1.3.2",
23
24
  "aws-sdk": "^2.544.0",
24
25
  "basic-auth": "^1.1.0",
25
26
  "body-parser": "^1.19.0",
@@ -79,7 +80,6 @@
79
80
  "request": "^2.88.0",
80
81
  "request-param": "^1.0.1",
81
82
  "response-time": "^2.3.2",
82
- "rwlock": "^5.0.0",
83
83
  "semver": "^5.7.1",
84
84
  "serve-favicon": "^2.5.0",
85
85
  "session-file-store": "^0.2.2",
package/temp/clusterlock/index.js CHANGED
@@ -5,7 +5,7 @@
5
5
  var path = require("path");
6
6
  var cluster = require("cluster");
7
7
 
8
- var ReadWriteLock = require("rwlock");
8
+ var AsyncLock = require('async-lock');
9
9
 
10
10
  var releaseFunctions = {};
11
11
 
@@ -18,7 +18,7 @@ var _setup = function() {
18
18
 
19
19
  if (cluster.isMaster)
20
20
  {
21
- var lock = new ReadWriteLock();
21
+ var lock = new AsyncLock();
22
22
 
23
23
  var _claim = function(message)
24
24
  {
@@ -26,7 +26,7 @@ var _setup = function() {
26
26
 
27
27
  var ticket = "ticket-" + message.id;
28
28
 
29
- lock.writeLock(key, function(releaseFn) {
29
+ lock.acquire(key, function(releaseFn) {
30
30
 
31
31
  releaseFunctions[ticket] = releaseFn;
32
32
 
package/temp/clusterlock/package.json CHANGED
@@ -13,7 +13,7 @@
13
13
  "shared"
14
14
  ],
15
15
  "devDependencies": {
16
- "rwlock": "^5.0.0"
16
+ "async-lock": "^1.3.2"
17
17
  },
18
18
  "maintainers": [
19
19
  {
package/util/cloudcms.js CHANGED
@@ -68,26 +68,23 @@ exports = module.exports = function()
68
68
  {
69
69
  if (!repositoryId)
70
70
  {
71
- callback({
71
+ return callback({
72
72
  "message": "Missing repositoryId in ensureContentDirectory()"
73
73
  });
74
- return;
75
74
  }
76
75
 
77
76
  if (!branchId)
78
77
  {
79
- callback({
78
+ return callback({
80
79
  "message": "Missing branchId in ensureContentDirectory()"
81
80
  });
82
- return;
83
81
  }
84
82
 
85
83
  if (!locale)
86
84
  {
87
- callback({
85
+ return callback({
88
86
  "message": "Missing locale in ensureContentDirectory()"
89
87
  });
90
- return;
91
88
  }
92
89
 
93
90
  var contentDirectoryPath = path.join(repositoryId, branchId, nodeId, locale);
@@ -148,21 +145,17 @@ exports = module.exports = function()
148
145
  if (err)
149
146
  {
150
147
  // nothing found
151
- callback({
148
+ return callback({
152
149
  "message": "Nothing cached on disk"
153
150
  });
154
-
155
- return;
156
151
  }
157
152
 
158
153
  if (!cacheInfoString)
159
154
  {
160
155
  // nothing found
161
- callback({
156
+ return callback({
162
157
  "message": "Nothing cached on disk"
163
158
  });
164
-
165
- return;
166
159
  }
167
160
 
168
161
  var invalidate = function () {
@@ -346,25 +339,40 @@ exports = module.exports = function()
346
339
  {
347
340
  if (attemptCount === maxAttemptsAllowed)
348
341
  {
349
- cb({
342
+ return cb({
350
343
  "message": "Maximum number of connection attempts exceeded(" + maxAttemptsAllowed + ")",
351
344
  "err": previousError
352
345
  });
353
-
354
- return;
355
346
  }
356
-
357
- contentStore.writeStream(filePath, function(err, tempStream) {
358
-
359
- if (err)
347
+
348
+ var failFast = function(contentStore, filePath, cb) {
349
+ var triggered = false;
350
+
351
+ return function(tempStream, err)
360
352
  {
353
+ // don't allow this to be called twice
354
+ if (triggered) {
355
+ return;
356
+ }
357
+
358
+ triggered = true;
359
+
361
360
  // ensure stream is closed
362
- closeWriteStream(tempStream);
363
-
361
+ if (tempStream) {
362
+ closeWriteStream(tempStream);
363
+ }
364
+
364
365
  // ensure cleanup
365
366
  return safeRemove(contentStore, filePath, function () {
366
367
  cb(err);
367
368
  });
369
+ };
370
+ }(contentStore, filePath, cb);
371
+
372
+ contentStore.writeStream(filePath, function(err, tempStream) {
373
+
374
+ if (err) {
375
+ return failFast(tempStream, err);
368
376
  }
369
377
 
370
378
  var cacheFilePath = toCacheFilePath(filePath);
@@ -398,18 +406,10 @@ exports = module.exports = function()
398
406
  {
399
407
  response.pipe(tempStream).on("close", function (err) {
400
408
 
401
- // TODO: not needed here?
402
- // ensure stream is closed
403
- // closeWriteStream(tempStream);
404
-
405
409
  if (err)
406
410
  {
407
411
  // some went wrong at disk io level?
408
- return safeRemove(contentStore, filePath, function () {
409
- cb({
410
- "message": "Failed to download: " + JSON.stringify(err)
411
- });
412
- });
412
+ return failFast(tempStream, err);
413
413
  }
414
414
 
415
415
  contentStore.existsFile(filePath, function (exists) {
@@ -418,55 +418,39 @@ exports = module.exports = function()
418
418
 
419
419
  // write cache file
420
420
  var cacheInfo = buildCacheInfo(response);
421
- if (cacheInfo)
422
- {
423
- contentStore.writeFile(cacheFilePath, JSON.stringify(cacheInfo, null, " "), function (err) {
424
-
425
- if (err)
426
- {
427
- // failed to write cache file, thus the whole thing is invalid
428
- safeRemove(contentStore, cacheFilePath, function () {
429
- safeRemove(contentStore, filePath, function () {
430
- cb({
431
- "message": "Failed to write cache file: " + cacheFilePath
432
- });
433
- });
421
+ if (!cacheInfo) {
422
+ return cb(null, filePath, null);
423
+ }
424
+
425
+ contentStore.writeFile(cacheFilePath, JSON.stringify(cacheInfo, null, " "), function (err) {
426
+
427
+ if (err)
428
+ {
429
+ // failed to write cache file, thus the whole thing is invalid
430
+ return safeRemove(contentStore, cacheFilePath, function () {
431
+ failFast(tempStream, {
432
+ "message": "Failed to write cache file: " + cacheFilePath + ", err: " + JSON.stringify(err)
434
433
  });
435
- }
436
- else
437
- {
438
- cb(null, filePath, cacheInfo);
439
- }
434
+ });
435
+ }
440
436
 
441
- });
442
- }
443
- else
444
- {
445
437
  cb(null, filePath, cacheInfo);
446
- }
438
+ });
447
439
  }
448
440
  else
449
441
  {
450
- //process.log("WARN: exists false, " + filePath);
451
-
452
442
  // for some reason, file wasn't found
453
443
  // roll back the whole thing
454
444
  safeRemove(contentStore, cacheFilePath, function () {
455
- safeRemove(contentStore, filePath, function () {
456
- cb({
457
- "message": "Failed to verify written cached file: " + filePath
458
- });
445
+ failFast(tempStream, {
446
+ "message": "Failed to verify written cached file: " + filePath
459
447
  });
460
448
  });
461
449
  }
462
450
  });
463
451
 
464
452
  }).on("error", function (err) {
465
-
466
- // ensure stream is closed
467
- closeWriteStream(tempStream);
468
-
469
- process.log("Pipe error: " + err);
453
+ failFast(tempStream, err);
470
454
  });
471
455
  }
472
456
  else
@@ -480,10 +464,7 @@ exports = module.exports = function()
480
464
  });
481
465
 
482
466
  response.on('end', function () {
483
-
484
- // ensure stream is closed
485
- closeWriteStream(tempStream);
486
-
467
+
487
468
  var afterCleanup = function () {
488
469
 
489
470
  // see if it is "invalid_token"
@@ -518,9 +499,11 @@ exports = module.exports = function()
518
499
  "code": response.statusCode,
519
500
  "body": body
520
501
  });
521
-
522
502
  };
523
-
503
+
504
+ // ensure stream is closed
505
+ closeWriteStream(tempStream);
506
+
524
507
  // clean things up
525
508
  safeRemove(contentStore, cacheFilePath, function () {
526
509
  safeRemove(contentStore, filePath, function () {
@@ -532,12 +515,8 @@ exports = module.exports = function()
532
515
  }
533
516
 
534
517
  }).on('error', function (e) {
535
-
536
- // ensure stream is closed
537
- closeWriteStream(tempStream);
538
-
539
- //process.log("_writeToDisk request timed out");
540
- //process.log(e)
518
+ failFast(tempStream, e);
519
+
541
520
  }).on('end', function (e) {
542
521
 
543
522
  // ensure stream is closed
@@ -548,9 +527,11 @@ exports = module.exports = function()
548
527
  tempStream.on("error", function (e) {
549
528
  process.log("Temp stream errored out");
550
529
  process.log(e);
530
+
531
+ failFast(tempStream, e);
551
532
 
552
533
  // ensure stream is closed
553
- closeWriteStream(tempStream);
534
+ //closeWriteStream(tempStream);
554
535
 
555
536
  });
556
537
  });
@@ -608,8 +589,7 @@ exports = module.exports = function()
608
589
  readFromDisk(contentStore, filePath, function (err, cacheInfo) {
609
590
 
610
591
  if (!err && cacheInfo) {
611
- callback(err, filePath, cacheInfo);
612
- return;
592
+ return callback(err, filePath, cacheInfo);
613
593
  }
614
594
 
615
595
  // either there was an error (in which case things were cleaned up)
@@ -631,13 +611,12 @@ exports = module.exports = function()
631
611
 
632
612
  if (err) {
633
613
  process.log("writeToDisk error, err: " + err.message + ", body: " + err.body);
634
- callback(err);
635
- }
636
- else {
637
- //process.log("Fetched: " + assetPath);
638
- //process.log("Retrieved from server: " + filePath);
639
- callback(null, filePath, cacheInfo);
614
+ return callback(err);
640
615
  }
616
+
617
+ //process.log("Fetched: " + assetPath);
618
+ //process.log("Retrieved from server: " + filePath);
619
+ callback(null, filePath, cacheInfo);
641
620
  });
642
621
  });
643
622
  };
@@ -749,6 +728,7 @@ exports = module.exports = function()
749
728
  writeToDisk(contentStore, gitana, uri, filePath, function (err, filePath, responseHeaders) {
750
729
 
751
730
  if (err) {
731
+
752
732
  if (err.code === 404) {
753
733
  return callback();
754
734
  }
@@ -1101,7 +1081,6 @@ exports = module.exports = function()
1101
1081
  _LOCK(contentStore, _lock_identifier(repositoryId, branchId, nodeId), function(err, releaseLockFn) {
1102
1082
 
1103
1083
  invalidateNode(contentStore, repositoryId, branchId, nodeId, function () {
1104
-
1105
1084
  invalidateNodePaths(contentStore, repositoryId, branchId, paths, function() {
1106
1085
 
1107
1086
  // release lock